From 6ad0f804dc209d8f741755756ac01c7b8b050cdf Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 3 Jun 2022 11:01:18 +0200 Subject: [PATCH] RED-4064: Always trigger reanalysis instead of surrounding text analysis when add a resize redaction --- .../service/ManualRedactionService.java | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 52cd24ee6..1c8db2d86 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -26,7 +26,6 @@ import com.google.common.hash.Hashing; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper; -import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; @@ -62,7 +61,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Manual import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; @@ -104,7 +102,8 @@ public class ManualRedactionService { private final HashFunction hashFunction = Hashing.murmur3_128(); - public List addAddRedaction(String dossierId, String fileId, List addRedactionRequests) { + public List addAddRedaction(String dossierId, String fileId, + List addRedactionRequests) { var response = new ArrayList(); @@ -117,7 +116,8 @@ public class ManualRedactionService { } validatePositions(fileId, addRedactionRequest); - String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); + String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8) + .toString(); addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); Long commentId = null; @@ -132,13 +132,12 @@ public class ManualRedactionService { response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); - }else { + } else { var manualTextRedactions = convert(response.stream() .map(r -> getAddRedaction(fileId, r.getAnnotationId())) @@ -146,7 +145,9 @@ public class ManualRedactionService { .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); if (!manualTextRedactions.isEmpty()) { - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); + ManualRedactions manualRedactions = ManualRedactions.builder() + .entriesToAdd(new HashSet<>(manualTextRedactions)) + .build(); addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); } @@ -155,7 +156,9 @@ public class ManualRedactionService { return response; } + private void validateDictionary(AddRedactionRequest addRedactionRequest) { + try { if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { throw new ConflictException("The entry you are trying to add is a stopword"); @@ -166,14 +169,20 @@ public class ManualRedactionService { } } + private void validatePositions(String fileId, AddRedactionRequest addRedactionRequest) { + var numberOfPages = fileStatusService.getStatus(fileId).getNumberOfPages(); - addRedactionRequest.getPositions().stream().filter(p -> p.getPage() > numberOfPages).findAny() + addRedactionRequest.getPositions() + .stream() + .filter(p -> p.getPage() > numberOfPages) + .findAny() .ifPresent((p) -> new BadRequestException("Invalid page found in the request")); } - public List addRemoveRedaction(String dossierId, String fileId, List removeRedactionRequests) { + public List addRemoveRedaction(String dossierId, String fileId, + List removeRedactionRequests) { var response = new ArrayList(); var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); @@ -202,14 +211,18 @@ public class ManualRedactionService { .stream() .filter(entry -> entry.getId().equals(removeRedactionRequest.getAnnotationId())) .findFirst(); - actionPerformed = actionPerformed || redactionLogEntryOptional.isPresent() && redactionLogEntryOptional.get().isHint(); + actionPerformed = actionPerformed || redactionLogEntryOptional.isPresent() && redactionLogEntryOptional.get() + .isHint(); } var removedFromDictionary = handleRemoveFromDictionary(redactionLog, dossier, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), false); actionPerformed = actionPerformed || removedFromDictionary; - response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); + response.add(ManualAddResponse.builder() + .annotationId(removeRedactionRequest.getAnnotationId()) + .commentId(commentId) + .build()); } } @@ -475,7 +488,6 @@ public class ManualRedactionService { var response = new ArrayList(); var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - boolean reprocess = false; RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true, true); for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) { @@ -487,18 +499,19 @@ public class ManualRedactionService { .stream() .filter(r -> r.getId().equals(resizeRedactionRequest.getAnnotationId())) .findFirst(); - if (redactionLogEntry.isPresent() && (redactionLogEntry.get().isDictionaryEntry() || redactionLogEntry.get().isDossierDictionaryEntry())) { + if (redactionLogEntry.isPresent() && (redactionLogEntry.get() + .isDictionaryEntry() || redactionLogEntry.get().isDossierDictionaryEntry())) { String typeId = redactionLogEntry.get().getType() + ":" + dossier.getDossierTemplateId(); if (redactionLogEntry.get().getValue().length() > resizeRedactionRequest.getValue().length()) { log.info("Remove old value '{}' from dictionary", redactionLogEntry.get().getValue()); - entryPersistenceService.deleteEntries(typeId, List.of(redactionLogEntry.get().getValue()), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY); + entryPersistenceService.deleteEntries(typeId, List.of(redactionLogEntry.get() + .getValue()), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY); } log.info("Add new value '{}' to dictionary", resizeRedactionRequest.getValue()); entryPersistenceService.addEntries(typeId, new HashSet<>(Collections.singleton(resizeRedactionRequest.getValue())), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY); - reprocess = true; } } @@ -507,16 +520,14 @@ public class ManualRedactionService { commentId = addComment(fileId, resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser()).getId(); } - var loaded = convert(getResizeRedaction(fileId, resizeRedactionRequest.getAnnotationId()), ManualResizeRedaction.class, new ManualResizeRedactionMapper()); - ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); + response.add(ManualAddResponse.builder() + .annotationId(resizeRedactionRequest.getAnnotationId()) + .commentId(commentId) + .build()); } - if (reprocess) { - reprocess(dossierId, fileId); - } + reprocess(dossierId, fileId); analysisFlagsCalculationService.calculateFlags(dossierId, fileId); @@ -545,7 +556,6 @@ public class ManualRedactionService { if (idRemoval.isRemoveFromDictionary()) { - var redactionLogEntry = redactionLogEntryOptional.get(); if (annotationStatus == AnnotationStatus.APPROVED) { @@ -627,8 +637,6 @@ public class ManualRedactionService { // These are marked as processed once surrounding text is computed ( TBD if this is correct ? ) manualRedactions.getEntriesToAdd() .forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter())); - manualRedactions.getResizeRedactions() - .forEach(e -> resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter())); } @@ -794,7 +802,8 @@ public class ManualRedactionService { } - private void addToDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) { + private void addToDictionary(String typeId, String value, String dossierId, String fileId, + DictionaryEntryType dictionaryEntryType) { try { log.debug("Adding entry: {} to {} for {} / {}", value, typeId, dossierId, fileId);