From ea5f79d67c12d2bb9d0fdf5aa6210f3471d34896 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 30 Aug 2023 13:02:40 +0200 Subject: [PATCH] RED-7317: Endpoint to change entity types of dict-based annotations * added undo functionality * fixed tests * added recategorize with dictionary tests --- .../controller/ManualRedactionController.java | 27 +- .../DeleteRemovedManualAddRedactions7.java | 2 +- .../v1/processor/service/CommentService.java | 128 +++ .../processor/service/FileStatusService.java | 3 +- ...anualRedactionDictionaryUpdateHandler.java | 250 ++++-- .../ManualRedactionProviderService.java | 2 +- .../ManualRedactionService.java | 767 +++++------------- .../ManualRedactionUndoService.java | 153 +++- ...ageRecategorizationPersistenceService.java | 1 + .../ManualImageRecategorizationMapper.java | 4 +- .../integration/service/TypeProvider.java | 13 +- .../v1/server/integration/tests/FileTest.java | 126 +-- .../tests/ManualRedactionTest.java | 77 +- .../ManualRequestWithAddToDictionary.java | 11 +- .../annotations/RecategorizationRequest.java | 1 - 15 files changed, 863 insertions(+), 702 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CommentService.java rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/{ => manualredactions}/ManualRedactionProviderService.java (99%) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java index 548846f37..8e4545d14 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java @@ -20,6 +20,7 @@ import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionUndoService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService; @@ -44,22 +45,26 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Remo import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @RestController @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualRedactionController implements ManualRedactionResource { - private static final String FILE_ID = "fileId"; - private static final String DOSSIER_ID = "dossierId"; - private static final String ANNOTATION_ID = "annotationId"; - private final ManualRedactionService manualRedactionService; - private final ManualRedactionUndoService manualRedactionUndoService; - private final DossierManagementService dossierManagementService; - private final AuditPersistenceService auditPersistenceService; - private final AccessControlService accessControlService; + final static String FILE_ID = "fileId"; + final static String DOSSIER_ID = "dossierId"; + final static String ANNOTATION_ID = "annotationId"; + ManualRedactionService manualRedactionService; + ManualRedactionUndoService manualRedactionUndoService; + DossierManagementService dossierManagementService; + AuditPersistenceService auditPersistenceService; + AccessControlService accessControlService; + CommentService commentService; @Override @@ -90,7 +95,7 @@ public class ManualRedactionController implements ManualRedactionResource { .message("Comment was removed.") .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId)) .build()); - manualRedactionService.deleteComment(dossierId, fileId, List.of(Long.valueOf(commentId))); + commentService.deleteComment(dossierId, fileId, List.of(Long.valueOf(commentId))); } @@ -107,7 +112,7 @@ public class ManualRedactionController implements ManualRedactionResource { @PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')") public Comments getComments(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) { - return manualRedactionService.getComments(fileId); + return commentService.getComments(fileId); } @@ -121,7 +126,7 @@ public class ManualRedactionController implements ManualRedactionResource { accessControlService.verifyFileIsNotApproved(dossierId, fileId); accessControlService.verifyUserIsReviewerOrApprover(dossierId, fileId); - var response = manualRedactionService.addComment(dossierId, + var response = commentService.addComment(dossierId, fileId, annotationId, CommentRequest.builder().user(KeycloakSecurity.getUserId()).text(addCommentRequest.getText()).build()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java index a072394e3..7ae908147 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/DeleteRemovedManualAddRedactions7.java @@ -9,7 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService; +import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CommentService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CommentService.java new file mode 100644 index 000000000..f5d5916ba --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/CommentService.java @@ -0,0 +1,128 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.CommentRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comments; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class CommentService { + + CommentPersistenceService commentPersistenceService; + FileStatusPersistenceService fileStatusPersistenceService; + FileStatusService fileStatusService; + RedactionLogService redactionLogService; + FileManagementStorageService fileManagementStorageService; + + + @Transactional + public void deleteComment(String dossierId, String fileId, List commentIds) { + + for (var commentId : commentIds) { + commentPersistenceService.softDelete(commentId, OffsetDateTime.now()); + } + // update indicator + fileStatusPersistenceService.updateHasComments(fileId, commentPersistenceService.fileHasComments(fileId)); + + // This is an ugly workaround, don't even look at it! + // Basically we should change how comments work and not merge them into the redaction log. + // if file is currently not analyzing, we can quickly change the redaction log, else we must wait for the analysis to finish. + if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) { + reprocess(dossierId, fileId); + return; + } + fileStatusService.setStatusProcessing(fileId); + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); + redactionLog.getRedactionLogEntry().forEach(entry -> entry.getComments().removeIf(comment -> commentIds.contains(comment.getId()))); + fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog); + fileStatusService.setStatusProcessed(fileId); + } + + + @Transactional + public Comments getComments(String fileId) { + + return new Comments(commentPersistenceService.findCommentsByFileID(fileId, false) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> MagicConverter.convert(entry.getValue(), Comment.class)))); + } + + + @Transactional + public CommentEntity addComment(String dossierId, String fileId, String annotationId, CommentRequest commentRequest) { + + CommentEntity createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser()); + + fileStatusPersistenceService.updateHasComments(fileId, true); + + // This is an ugly workaround, don't even look at it! + // Basically we should change how comments work and not merge them into the redaction log. + if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) { + reprocess(dossierId, fileId); + return createdComment; + } + fileStatusService.setStatusProcessing(fileId); + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); + redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getId().equals(annotationId)) + .forEach(entry -> entry.getComments().add(MagicConverter.convert(createdComment, RedactionLogComment.class))); + fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog); + fileStatusService.setStatusProcessed(fileId); + return createdComment; + } + + + public Long addCommentAndGetId(String fileId, String annotationId, String comment, String user) { + + if (comment == null) { + return null; + } + return addComment(fileId, annotationId, comment, user).getId(); + } + + + private CommentEntity addComment(String fileId, String annotationId, String comment, String user) { + + if (comment == null) { + return null; + } + + return commentPersistenceService.insert(CommentEntity.builder() + .text(comment) + .fileId(fileId) + .annotationId(annotationId) + .user(user) + .date(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)) + .build()); + } + + + private void reprocess(String dossierId, String fileId) { + + fileStatusService.setStatusReprocessForManual(dossierId, fileId, true); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index d14a7d197..840becd04 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service; import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -13,7 +12,6 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; @@ -25,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.model.NerServic import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse; import com.iqser.red.service.persistence.management.v1.processor.model.image.ImageServiceRequest; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestFactory; +import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java index 14d26e786..97a5e84b8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java @@ -2,18 +2,26 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import org.springframework.stereotype.Service; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.StopwordService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithAddToDictionary; @@ -23,71 +31,231 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import feign.FeignException; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualRedactionDictionaryUpdateHandler { - private final DictionaryManagementService dictionaryManagementService; - private final AddRedactionPersistenceService addRedactionPersistenceService; - private final FileStatusPersistenceService fileStatusPersistenceService; - private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; - private final DossierPersistenceService dossierPersistenceService; + DictionaryManagementService dictionaryManagementService; + DictionaryPersistenceService dictionaryPersistenceService; + FileStatusPersistenceService fileStatusPersistenceService; + ResizeRedactionPersistenceService resizeRedactionPersistenceService; + DossierPersistenceService dossierPersistenceService; + StopwordService stopwordService; - public Set handleAddToDictionaryAndReturnModifiedTypeIds(String fileId, String annotationId, ManualRequestWithAddToDictionary manualRequestWithAddToDictionary) { - - if (!manualRequestWithAddToDictionary.isApproved()) { - return null; - } + public Set handleAddToDictionaryAndReturnModifiedTypeIds(String fileId, String value, ManualRequestWithAddToDictionary manualRequestWithAddToDictionary) { if (!manualRequestWithAddToDictionary.isAddToDictionary()) { - addRedactionPersistenceService.updateStatus(fileId, annotationId, manualRequestWithAddToDictionary.getStatus(), false, null); - return null; + return Collections.emptySet(); } Set typeIdsOfModifiedDictionaries = new HashSet<>(); if (manualRequestWithAddToDictionary.isAddToAllDossiers()) { - var dictionaryEntriesToUnDelete = dictionaryManagementService.getAllEntriesInDossierTemplate(manualRequestWithAddToDictionary.getDossierTemplateTypeId(), - manualRequestWithAddToDictionary.getValue()); + List dictionaryEntriesToUnDelete = dictionaryManagementService.getAllEntriesInDossierTemplate(manualRequestWithAddToDictionary.getDossierTemplateTypeId(), + value); dictionaryEntriesToUnDelete.forEach(entry -> { typeIdsOfModifiedDictionaries.add(entry.getTypeId()); - addToDictionary(entry.getTypeId(), manualRequestWithAddToDictionary.getValue(), manualRequestWithAddToDictionary.getDossierId(), fileId, manualRequestWithAddToDictionary.getDictionaryEntryType()); + addToDictionary(entry.getTypeId(), + value, + manualRequestWithAddToDictionary.getDossierId(), + fileId, + manualRequestWithAddToDictionary.getDictionaryEntryType()); }); addToDictionary(manualRequestWithAddToDictionary.getDossierTemplateTypeId(), - manualRequestWithAddToDictionary.getValue(), + value, manualRequestWithAddToDictionary.getDossierId(), fileId, manualRequestWithAddToDictionary.getDictionaryEntryType()); typeIdsOfModifiedDictionaries.add(manualRequestWithAddToDictionary.getDossierTemplateTypeId()); - return typeIdsOfModifiedDictionaries; } addToDictionary(manualRequestWithAddToDictionary.getDossierTemplateTypeId() + ":" + manualRequestWithAddToDictionary.getDossierId(), - manualRequestWithAddToDictionary.getValue(), + value, manualRequestWithAddToDictionary.getDossierId(), fileId, manualRequestWithAddToDictionary.getDictionaryEntryType()); typeIdsOfModifiedDictionaries.add(manualRequestWithAddToDictionary.getDossierTemplateTypeId() + ":" + manualRequestWithAddToDictionary.getDossierId()); return typeIdsOfModifiedDictionaries; + } + + public Set handleRemoveFromDictionaryAndReturnModifiedTypeIds(RedactionLogEntry redactionLogEntry, + String fileId, + String dossierId, + String dossierTemplateId, + ManualRequestWithRemoveFromDictionary manualRequestWithRemoveFromDictionary) { + + if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary()) { + + return Collections.emptySet(); + } + + Set typeIdsOfModifiedDictionaries = new HashSet<>(); + if (manualRequestWithRemoveFromDictionary.isRemoveFromAllDossiers()) { + var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(), dossierTemplateId), + redactionLogEntry.getValue()); + dictionaryEntriesToRemove.forEach(entry -> { + typeIdsOfModifiedDictionaries.add(entry.getTypeId()); + removeFromDictionary(entry.getTypeId(), entry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY); + }); + } else { + typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId)); + removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY); + } + + // This is needed to remove resizeRedactions with addToDictionary. + removeResizeRedactionsWithAddToDictionary(dossierTemplateId, redactionLogEntry.getValue()); + + return typeIdsOfModifiedDictionaries; + } + + + public Set updateDictionaryForResizeRedactions(String dossierId, String fileId, ManualResizeRedactionEntity resizeRedaction, RedactionLogEntry redactionLogEntry) { + + if (resizeRedaction.getUpdateDictionary() != null && resizeRedaction.getUpdateDictionary() && resizeRedaction.getStatus() + .equals(AnnotationStatus.APPROVED) && (redactionLogEntry.isDictionaryEntry() || redactionLogEntry.isDossierDictionaryEntry())) { + + var dossier = dossierPersistenceService.findByDossierId(dossierId); + + var typeId = buildTypeId(redactionLogEntry, resizeRedaction, dossier); + var newValue = resizeRedaction.getValue(); + var oldValue = redactionLogEntry.getValue(); + var dictionaryEntryType = getDictionaryEntryType(redactionLogEntry); + + boolean isShrinking = oldValue != null && oldValue.length() > newValue.length(); + + Set typeIdsOfModifiedDictionaries = new HashSet<>(); + + if (isShrinking) { + log.info("Remove old value '{}' from dictionary", oldValue); + removeFromDictionary(typeId, oldValue, dossierId, fileId, dictionaryEntryType); + typeIdsOfModifiedDictionaries.add(typeId); + + if (resizeRedaction.isAddToAllDossiers() && redactionLogEntry.isDictionaryEntry()) { + String dossierTemplateId = dossier.getDossierTemplateId(); + var dossiersOfThisDossierTemplate = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); + var type = redactionLogEntry.getType(); + dossiersOfThisDossierTemplate.forEach(dossierEntity -> { + var typeIdOfDossierEntity = toTypeId(type, dossierTemplateId, dossierEntity.getId()); + removeFromDictionary(typeIdOfDossierEntity, oldValue, dossierId, fileId, dictionaryEntryType); + typeIdsOfModifiedDictionaries.add(typeIdOfDossierEntity); + }); + } + + } + + log.info("Add new value '{}' to dictionary", newValue); + addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType); + typeIdsOfModifiedDictionaries.add(typeId); + + return typeIdsOfModifiedDictionaries; + } + return Collections.emptySet(); + } + + + private String buildTypeId(RedactionLogEntry redactionLogEntry, ManualResizeRedactionEntity resizeRedaction, DossierEntity dossier) { + + if (resizeRedaction.isAddToAllDossiers()) { + return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId()); + } else { + return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()); + } + + } + + + private DictionaryEntryType getDictionaryEntryType(RedactionLogEntry redactionLogEntry) { + + if (redactionLogEntry.isRecommendation() && redactionLogEntry.isFalsePositive()) { + return DictionaryEntryType.FALSE_RECOMMENDATION; + } else if (redactionLogEntry.isFalsePositive()) { + return DictionaryEntryType.FALSE_POSITIVE; + } else { + return DictionaryEntryType.ENTRY; + } } public boolean revertAddToDictionary(String fileId, String dossierId, ManualRedactionEntryEntity manualRedactionToRevert) { - if (!manualRedactionToRevert.getStatus().equals(AnnotationStatus.APPROVED) || manualRedactionToRevert.isAddToDictionary()) { + if (!manualRedactionToRevert.isAddToDictionary()) { return false; } - manualRedactionToRevert.getTypeIdsOfModifiedDictionaries().forEach(typeId -> { - removeFromDictionary(typeId, manualRedactionToRevert.getValue(), dossierId, fileId, manualRedactionToRevert.getDictionaryEntryType()); - }); + revertAddToDictionary(manualRedactionToRevert.getValue(), + manualRedactionToRevert.getDictionaryEntryType(), + fileId, + dossierId, + manualRedactionToRevert.getTypeIdsOfModifiedDictionaries()); return true; } + public void revertAddToDictionary(String value, DictionaryEntryType dictionaryEntryType, String fileId, String dossierId, Set typeIdsOfModifiedDictionaries) { + + typeIdsOfModifiedDictionaries.forEach(typeId -> { + removeFromDictionary(typeId, value, dossierId, fileId, dictionaryEntryType); + }); + } + + + public boolean revertRemoveFromDictionary(String value, String dossierId, String fileId, IdRemovalEntity idRemoval) { + + if (!idRemoval.isRemoveFromDictionary()) { + return false; + } + + revertRemoveFromDictionary(value, dossierId, fileId, idRemoval.getTypeIdsOfModifiedDictionaries()); + return true; + } + + + public void revertRemoveFromDictionary(String value, String dossierId, String fileId, Set typeIdsOfModifiedDictionaries) { + + typeIdsOfModifiedDictionaries.forEach(changedTypeId -> addToDictionary(changedTypeId, value, dossierId, fileId, DictionaryEntryType.ENTRY)); + } + + + public void validateDictionariesForDelete(ManualRequestWithRemoveFromDictionary request, RedactionLogEntry redactionLogEntry, String dossierTemplateId) { + + if (request.isRemoveFromDictionary()) { + var dossierTemplateTypeId = toTypeId(redactionLogEntry.getType(), dossierTemplateId); + dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(dossierTemplateTypeId, request.isRemoveFromDictionary(), request.isRemoveFromAllDossiers()); + } + } + + + public void validateDictionariesForAdd(ManualRequestWithAddToDictionary addRedactionRequest, String value) { + + if (addRedactionRequest.isAddToDictionary()) { + if (addRedactionRequest.isAddToAllDossiers()) { + // validate add to dossier template dictionaries + dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(addRedactionRequest.getDossierTemplateTypeId()); + } + validateDictionary(addRedactionRequest, value); + } + } + + + private void validateDictionary(ManualRequestWithAddToDictionary addRedactionRequest, String value) { + + try { + if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(value)) { + throw new ConflictException("The entry you are trying to add is a stopword"); + } + dictionaryPersistenceService.getType(addRedactionRequest.getDossierTemplateTypeId()); + } catch (NotFoundException e) { + throw new BadRequestException("Invalid type: " + addRedactionRequest.getDossierTemplateTypeId()); + } + } + + private void addToDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) { try { @@ -110,41 +278,6 @@ public class ManualRedactionDictionaryUpdateHandler { } - public Set handleRemoveFromDictionaryAndReturnModifiedTypeIds(RedactionLogEntry redactionLogEntry, - String fileId, - String dossierId, - String dossierTemplateId, - ManualRequestWithRemoveFromDictionary manualRequestWithRemoveFromDictionary) { - - if (!manualRequestWithRemoveFromDictionary.isApproved()) { - return null; - } - - if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary()) { - - return null; - } - - Set typeIdsOfModifiedDictionaries = new HashSet<>(); - if (manualRequestWithRemoveFromDictionary.isRemoveFromAllDossiers()) { - var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(), dossierTemplateId), - redactionLogEntry.getValue()); - dictionaryEntriesToRemove.forEach(entry -> { - typeIdsOfModifiedDictionaries.add(entry.getTypeId()); - removeFromDictionary(entry.getTypeId(), entry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY); - }); - } else { - typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId)); - removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY); - } - - // This is needed to remove resizeRedactions with addToDictionary. - removeResizeRedactionsWithAddToDictionary(dossierTemplateId, redactionLogEntry.getValue()); - - return typeIdsOfModifiedDictionaries; - } - - private void removeResizeRedactionsWithAddToDictionary(String dossierTemplateId, String redactionLogEntryValue) { var resizeRedactionsWithSameValue = resizeRedactionPersistenceService.findByAnnotationStatusAndValue(AnnotationStatus.APPROVED, redactionLogEntryValue); @@ -157,5 +290,4 @@ public class ManualRedactionDictionaryUpdateHandler { }); } - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionProviderService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java similarity index 99% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionProviderService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java index 54511c53d..862cb9126 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionProviderService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionProviderService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; +package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions; import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java index c5f64a39d..2318b92f5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java @@ -1,44 +1,24 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions; -import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; - import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; -import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService; +import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService; -import com.iqser.red.service.persistence.management.v1.processor.service.StopwordService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; @@ -47,53 +27,44 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.BaseManualRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.CommentRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comments; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithAddToDictionary; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import feign.FeignException; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualRedactionService { - private final DossierPersistenceService dossierPersistenceService; - private final AddRedactionPersistenceService addRedactionPersistenceService; - private final RemoveRedactionPersistenceService removeRedactionPersistenceService; - private final ForceRedactionPersistenceService forceRedactionPersistenceService; - private final CommentPersistenceService commentPersistenceService; - private final FileStatusPersistenceService fileStatusPersistenceService; - private final DictionaryPersistenceService dictionaryPersistenceService; - private final FileManagementStorageService fileManagementStorageService; - private final ImageRecategorizationPersistenceService recategorizationPersistenceService; - private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; - private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; - private final FileStatusService fileStatusService; - private final ManualRedactionProviderService manualRedactionProviderService; - private final AnalysisFlagsCalculationService analysisFlagsCalculationService; - private final StopwordService stopwordService; - private final RedactionLogService redactionLogService; - private final DictionaryManagementService dictionaryManagementService; - private final HashFunction hashFunction = Hashing.murmur3_128(); - private final ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler; + DossierPersistenceService dossierPersistenceService; + AddRedactionPersistenceService addRedactionPersistenceService; + RemoveRedactionPersistenceService removeRedactionPersistenceService; + ForceRedactionPersistenceService forceRedactionPersistenceService; + CommentService commentService; + ImageRecategorizationPersistenceService recategorizationPersistenceService; + LegalBasisChangePersistenceService legalBasisChangePersistenceService; + ResizeRedactionPersistenceService resizeRedactionPersistenceService; + FileStatusService fileStatusService; + ManualRedactionProviderService manualRedactionProviderService; + AnalysisFlagsCalculationService analysisFlagsCalculationService; + RedactionLogService redactionLogService; + DictionaryManagementService dictionaryManagementService; + HashFunction hashFunction = Hashing.murmur3_128(); + ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler; @Transactional @@ -104,24 +75,22 @@ public class ManualRedactionService { dossierPersistenceService.getAndValidateDossier(dossierId); for (AddRedactionRequest addRedactionRequest : addRedactionRequests) { - validateDictionaries(addRedactionRequest); + manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue()); validatePositions(fileId, addRedactionRequest); String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); - Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, - annotationId, - addRedactionRequest); + Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, addRedactionRequest.getValue(), addRedactionRequest); addRedactionPersistenceService.updateStatus(fileId, annotationId, addRedactionRequest.getStatus(), - typeIdsOfModifiedDictionaries != null, + !typeIdsOfModifiedDictionaries.isEmpty(), typeIdsOfModifiedDictionaries); - Long commentId = addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()); + Long commentId = commentService.addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()); response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } @@ -136,28 +105,202 @@ public class ManualRedactionService { } - private void validateDictionaries(ManualRequestWithAddToDictionary addRedactionRequest) { + public List addRemoveRedaction(String dossierId, String fileId, List removeRedactionRequests) { - if (addRedactionRequest.isAddToDictionary()) { - if (addRedactionRequest.isAddToAllDossiers()) { - // validate add to dossier template dictionaries - dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(addRedactionRequest.getDossierTemplateTypeId()); + var response = new ArrayList(); + var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossier.getId(), fileId); + + var requiresReAnalysis = false; + var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId); + + //validate removing from dossier template dictionary + + for (RemoveRedactionRequest removeRedactionRequest : removeRedactionRequests) { + RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()); + manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest, redactionLogEntry, dossier.getDossierTemplateId()); + + removeRedactionPersistenceService.insert(fileId, removeRedactionRequest); + + if (manualAddRedactionsContains(manualRedactions, removeRedactionRequest.getAnnotationId())) { + log.info("hard delete ManualRedactions for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId()); + manualRedactionProviderService.hardDeleteManualRedactions(fileId, removeRedactionRequest.getAnnotationId()); + requiresReAnalysis = true; + continue; } - validateDictionary(addRedactionRequest); + + log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId()); + + Long commentId = commentService.addCommentAndGetId(fileId, + removeRedactionRequest.getAnnotationId(), + removeRedactionRequest.getComment(), + removeRedactionRequest.getUser()); + + Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(// + redactionLogEntry,// + fileId,// + dossierId, // + dossier.getDossierTemplateId(),// + removeRedactionRequest); + + boolean removedFromDictionary = !typeIdsOfModifiedDictionaries.isEmpty(); + + removeRedactionPersistenceService.updateStatus(fileId, + removeRedactionRequest.getAnnotationId(), + removeRedactionRequest.getStatus(), + removedFromDictionary, + typeIdsOfModifiedDictionaries); + + requiresReAnalysis = requiresReAnalysis || removedFromDictionary; + + response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); } + + if (requiresReAnalysis) { + reprocess(dossierId, fileId); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + + return response; } - private void validateDictionary(ManualRequestWithAddToDictionary addRedactionRequest) { + @Transactional + public List addForceRedaction(String dossierId, String fileId, List forceRedactionRequests) { - try { - if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { - throw new ConflictException("The entry you are trying to add is a stopword"); - } - dictionaryPersistenceService.getType(addRedactionRequest.getDossierTemplateTypeId()); - } catch (NotFoundException e) { - throw new BadRequestException("Invalid type: " + addRedactionRequest.getDossierTemplateTypeId()); + var response = new ArrayList(); + dossierPersistenceService.getAndValidateDossier(dossierId); + var requiresReanalysis = false; + + for (var forceRedactionRequest : forceRedactionRequests) { + forceRedactionPersistenceService.insert(fileId, forceRedactionRequest); + + Long commentId = commentService.addCommentAndGetId(fileId, + forceRedactionRequest.getAnnotationId(), + forceRedactionRequest.getComment(), + forceRedactionRequest.getUser()); + + requiresReanalysis = requiresReanalysis || forceRedactionRequest.isApproved(); + response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build()); } + + if (requiresReanalysis) { + reprocess(dossierId, fileId); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + + return response; + } + + + @Transactional + public List addLegalBasisChange(String dossierId, String fileId, List legalBasisChangeRequests) { + + var response = new ArrayList(); + dossierPersistenceService.getAndValidateDossier(dossierId); + + for (var legalBasisChangeRequest : legalBasisChangeRequests) { + legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest); + + Long commentId = commentService.addCommentAndGetId(fileId, + legalBasisChangeRequest.getAnnotationId(), + legalBasisChangeRequest.getComment(), + legalBasisChangeRequest.getUser()); + + response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build()); + } + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + + return response; + } + + + @Transactional + public List addRecategorization(String dossierId, String fileId, List recategorizationRequests) { + + var response = new ArrayList(); + var requiresReanalysis = false; + var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); + for (var recategorizationRequest : recategorizationRequests) { + RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, recategorizationRequest.getAnnotationId()); + manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(recategorizationRequest, redactionLogEntry.getValue()); + manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(recategorizationRequest, redactionLogEntry, dossier.getDossierTemplateId()); + + recategorizationPersistenceService.insert(fileId, recategorizationRequest); + + Set typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, redactionLogEntry.getValue(), recategorizationRequest); + Set typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(redactionLogEntry, + fileId, + recategorizationRequest.getDossierId(), + dossier.getDossierTemplateId(), + recategorizationRequest); + + recategorizationPersistenceService.updateStatus(fileId, + recategorizationRequest.getAnnotationId(), + AnnotationStatus.APPROVED, + typeIdsOfDictionariesWithAdd != null, + typeIdsOfDictionariesWithAdd, + typeIdsOfDictionariesWithDelete); + + Long commentId = commentService.addCommentAndGetId(fileId, + recategorizationRequest.getAnnotationId(), + recategorizationRequest.getComment(), + recategorizationRequest.getUser()); + + requiresReanalysis = true; + + response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build()); + } + if (requiresReanalysis) { + reprocess(dossierId, fileId); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + + return response; + } + + + @Transactional + public List addResizeRedaction(String dossierId, String fileId, List resizeRedactionRequests) { + + List response = new ArrayList<>(); + + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); + + for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) { + + var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest); + + if (resizeRedactionRequest.getComment() != null) { + Long commentId = commentService.addCommentAndGetId(fileId, + resizeRedactionRequest.getAnnotationId(), + resizeRedactionRequest.getComment(), + resizeRedactionRequest.getUser()); + response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); + } + + Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.updateDictionaryForResizeRedactions(dossierId, + fileId, + resizeRedaction, + getRedactionLogEntry(redactionLog, resizeRedaction.getId().getAnnotationId())); + + resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(), + resizeRedaction.getId().getAnnotationId(), + resizeRedaction.getStatus(), + typeIdsOfModifiedDictionaries); + } + + if (resizeRedactionRequests.stream().anyMatch(BaseManualRequest::isApproved)) { + reprocess(dossierId, fileId); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + + return response; } @@ -168,43 +311,12 @@ public class ManualRedactionService { } - private Long addCommentAndGetId(String fileId, String annotationId, String comment, String user) { - - if (comment == null) { - return null; - } - return addComment(fileId, annotationId, comment, user).getId(); - } - - - private CommentEntity addComment(String fileId, String annotationId, String comment, String user) { - - if (comment == null) { - return null; - } - - return commentPersistenceService.insert(CommentEntity.builder() - .text(comment) - .fileId(fileId) - .annotationId(annotationId) - .user(user) - .date(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)) - .build()); - } - - private void reprocess(String dossierId, String fileId) { fileStatusService.setStatusReprocessForManual(dossierId, fileId, true); } - public ManualRedactionEntryEntity getAddRedaction(String fileId, String annotationId) { - - return addRedactionPersistenceService.findAddRedaction(fileId, annotationId); - } - - private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) { try { @@ -227,111 +339,12 @@ public class ManualRedactionService { } - public List addRemoveRedaction(String dossierId, String fileId, List removeRedactionRequests) { - - var response = new ArrayList(); - var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId); - - var requiresReAnalysis = false; - var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId); - - //validate removing from dossier template dictionary - removeRedactionRequests.forEach(request -> { - if (request.isRemoveFromDictionary()) { - RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, request.getAnnotationId()); - var dossierTemplateTypeId = toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId()); - dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(dossierTemplateTypeId, - request.isRemoveFromDictionary(), - request.isRemoveFromAllDossiers()); - } - }); - - for (RemoveRedactionRequest removeRedactionRequest : removeRedactionRequests) { - - if (manualAddRedactionsContains(manualRedactions, removeRedactionRequest.getAnnotationId()) && removeRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED)) { - log.info("hard delete ManualRedactions for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId()); - manualRedactionProviderService.hardDeleteManualRedactions(fileId, removeRedactionRequest.getAnnotationId()); - requiresReAnalysis = true; - continue; - } - - log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId()); - IdRemoval idRemoval = MagicConverter.convert(removeRedactionPersistenceService.insert(fileId, removeRedactionRequest), IdRemoval.class); - - Long commentId = addCommentAndGetId(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()); - boolean matchingEntryFound = false; - if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) { - try { - getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()); - matchingEntryFound = true; - } catch (NotFoundException e) { - log.warn("No matching entry found in redaction log for annotation id {}", removeRedactionRequest.getAnnotationId()); - } - - requiresReAnalysis = requiresReAnalysis || matchingEntryFound; - } - - Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(// - getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()),// - dossierId, // - dossier.getDossierTemplateId(),// - fileId, // - removeRedactionRequest); - - boolean removedFromDictionary = typeIdsOfModifiedDictionaries != null; - removeRedactionPersistenceService.updateStatus(fileId, - removeRedactionRequest.getAnnotationId(), - removeRedactionRequest.getStatus(), - removedFromDictionary, - typeIdsOfModifiedDictionaries); - - if (!matchingEntryFound && !removedFromDictionary && idRemoval.isApproved()) { - removeRedactionPersistenceService.markAsProcessed(idRemoval); - } - - requiresReAnalysis = requiresReAnalysis || removedFromDictionary; - - response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); - } - - if (requiresReAnalysis) { - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - - return response; - } - - private boolean manualAddRedactionsContains(ManualRedactions manualRedactions, String annotationId) { return manualRedactions.getEntriesToAdd().stream().anyMatch(m -> annotationId.equals(m.getAnnotationId())); } - private boolean revertRemoveFromDictionary(String value, DossierEntity dossier, String fileId, IdRemovalEntity idRemoval) { - - String annotationId = idRemoval.getId().getAnnotationId(); - - if (!idRemoval.getStatus().equals(AnnotationStatus.APPROVED)) { - return false; - } - if (!idRemoval.isRemoveFromDictionary()) { - removeRedactionPersistenceService.updateStatus(fileId, annotationId, idRemoval.getStatus(), false, Collections.emptySet()); - return false; - } - - idRemoval.getTypeIdsOfModifiedDictionaries().forEach(changedTypeId -> addToDictionary(changedTypeId, value, dossier.getId(), fileId, DictionaryEntryType.ENTRY)); - removeRedactionPersistenceService.updateStatus(fileId, annotationId, idRemoval.getStatus(), true, Collections.emptySet()); - return true; - } - - - - - private RedactionLogEntry getRedactionLogEntry(RedactionLog redactionLog, String annotationId) { return redactionLog.getRedactionLogEntry() @@ -342,355 +355,6 @@ public class ManualRedactionService { } - - @Transactional - public List addForceRedaction(String dossierId, String fileId, List forceRedactionRequests) { - - var response = new ArrayList(); - dossierPersistenceService.getAndValidateDossier(dossierId); - var requiresReanalysis = false; - - for (var forceRedactionRequest : forceRedactionRequests) { - forceRedactionPersistenceService.insert(fileId, forceRedactionRequest); - - Long commentId = addCommentAndGetId(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()); - - requiresReanalysis = requiresReanalysis || forceRedactionRequest.isApproved(); - response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build()); - } - - if (requiresReanalysis) { - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - - return response; - } - - - @Transactional - public List addLegalBasisChange(String dossierId, String fileId, List legalBasisChangeRequests) { - - var response = new ArrayList(); - dossierPersistenceService.getAndValidateDossier(dossierId); - - for (var legalBasisChangeRequest : legalBasisChangeRequests) { - legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest); - - Long commentId = addCommentAndGetId(fileId, legalBasisChangeRequest.getAnnotationId(), legalBasisChangeRequest.getComment(), legalBasisChangeRequest.getUser()); - - response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build()); - } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - - return response; - } - - - @Transactional - public List addRecategorization(String dossierId, String fileId, List recategorizationRequests) { - - var response = new ArrayList(); - var requiresReanalysis = false; - var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId); - for (var recategorizationRequest : recategorizationRequests) { - validateDictionaries(recategorizationRequest); - - recategorizationPersistenceService.insert(fileId, recategorizationRequest); - - Set typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, - recategorizationRequest.getAnnotationId(), - recategorizationRequest); - - Set typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(// - getRedactionLogEntry(redactionLog, recategorizationRequest.getAnnotationId()),// - recategorizationRequest.getDossierId(),// - dossier.getDossierTemplateId(),// - fileId,// - recategorizationRequest); - - recategorizationPersistenceService.updateStatus(fileId, - recategorizationRequest.getAnnotationId(), - AnnotationStatus.APPROVED, - typeIdsOfDictionariesWithAdd != null, - typeIdsOfDictionariesWithAdd, - typeIdsOfDictionariesWithDelete); - - Long commentId = addCommentAndGetId(fileId, recategorizationRequest.getAnnotationId(), recategorizationRequest.getComment(), recategorizationRequest.getUser()); - - requiresReanalysis = requiresReanalysis || recategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED); - - response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).build()); - } - if (requiresReanalysis) { - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - - return response; - } - - - public CommentEntity addComment(String dossierId, String fileId, String annotationId, CommentRequest commentRequest) { - - CommentEntity createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser()); - - fileStatusPersistenceService.updateHasComments(fileId, true); - if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) { - reprocess(dossierId, fileId); - return createdComment; - } - fileStatusService.setStatusProcessing(fileId); - RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); - redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getId().equals(annotationId)) - .forEach(entry -> entry.getComments().add(MagicConverter.convert(createdComment, RedactionLogComment.class))); - fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog); - fileStatusService.setStatusProcessed(fileId); - return createdComment; - } - - - @Transactional - public void deleteAddRedaction(String dossierId, String fileId, List annotationIds) { - - var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var requiresReanalysis = false; - - for (String annotationId : annotationIds) { - - ManualRedactionEntryEntity addRedaction = getAddRedaction(fileId, annotationId); - - boolean dictionaryChanged = manualRedactionDictionaryUpdateHandler.revertAddToDictionary(fileId, dossier.getId(), addRedaction); - requiresReanalysis = requiresReanalysis || dictionaryChanged; - addRedactionPersistenceService.updateStatus(fileId, annotationId, AnnotationStatus.APPROVED, false, null); - addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); - } - - if (requiresReanalysis) { - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - } - - - @Transactional - public void deleteRemoveRedaction(String dossierId, String fileId, List annotationIds) { - - var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var requiresReanalysis = false; - RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); - - for (String annotationId : annotationIds) { - - IdRemovalEntity removeRedaction = getRemoveRedaction(fileId, annotationId); - String originalValue = getRedactionLogEntry(redactionLog, annotationId).getValue(); - boolean dictionaryChanged = revertRemoveFromDictionary(originalValue, dossier, fileId, removeRedaction); - - requiresReanalysis = requiresReanalysis || dictionaryChanged; - removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); - } - - if (requiresReanalysis) { - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - } - - - private IdRemovalEntity getRemoveRedaction(String fileId, String annotationId) { - - return removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId); - } - - - @Transactional - public void deleteForceRedaction(String dossierId, String fileId, List annotationIds) { - - var now = OffsetDateTime.now(); - - for (var annotationId : annotationIds) { - forceRedactionPersistenceService.softDelete(fileId, annotationId, now); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - } - - - @Transactional - public void deleteLegalBasisChange(String dossierId, String fileId, List annotationIds) { - - for (var annotationId : annotationIds) { - legalBasisChangePersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - } - - - @Transactional - public void deleteRecategorization(String dossierId, String fileId, List annotationIds) { - - var requiresReanalysis = false; - for (var annotationId : annotationIds) { - var imageRecategorization = getRecategorization(fileId, annotationId); - - recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); - - requiresReanalysis = requiresReanalysis || !AnnotationStatus.REQUESTED.equals(imageRecategorization.getStatus()); - } - - if (requiresReanalysis) { - reprocess(dossierId, fileId); - } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - } - - - private ManualImageRecategorizationEntity getRecategorization(String fileId, String annotationId) { - - return recategorizationPersistenceService.findRecategorization(fileId, annotationId); - } - - - @Transactional - public void deleteComment(String dossierId, String fileId, List commentIds) { - - for (var commentId : commentIds) { - commentPersistenceService.softDelete(commentId, OffsetDateTime.now()); - } - // update indicator - fileStatusPersistenceService.updateHasComments(fileId, commentPersistenceService.fileHasComments(fileId)); - // if file is currently not analyzing, we can quickly change the redaction log, else we must wait for the analysis to finish. - if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) { - reprocess(dossierId, fileId); - return; - } - fileStatusService.setStatusProcessing(fileId); - RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); - redactionLog.getRedactionLogEntry().forEach(entry -> entry.getComments().removeIf(comment -> commentIds.contains(comment.getId()))); - fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog); - fileStatusService.setStatusProcessed(fileId); - } - - - @Transactional - public void deleteResizeRedaction(String dossierId, String fileId, List annotationIds) { - - OffsetDateTime now = OffsetDateTime.now(); - for (var annotationId : annotationIds) { - resizeRedactionPersistenceService.softDelete(fileId, annotationId, now); - } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - } - - - @Transactional - public List addResizeRedaction(String dossierId, String fileId, List resizeRedactionRequests) { - - List response = new ArrayList<>(); - - RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); - - for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) { - - var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest); - - if (resizeRedactionRequest.getComment() != null) { - Long commentId = addCommentAndGetId(fileId, resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); - } - - updateDictionaryForResizeRedactions(dossierId, fileId, resizeRedaction, redactionLog); - } - - if (resizeRedactionRequests.stream().anyMatch(BaseManualRequest::isApproved)) { - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - - return response; - } - - - private void updateDictionaryForResizeRedactions(String dossierId, String fileId, ManualResizeRedactionEntity resizeRedaction, RedactionLog redactionLog) { - - RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, resizeRedaction.getId().getAnnotationId()); - - if (resizeRedaction.getUpdateDictionary() != null && resizeRedaction.getUpdateDictionary() && resizeRedaction.getStatus() - .equals(AnnotationStatus.APPROVED) && (redactionLogEntry.isDictionaryEntry() || redactionLogEntry.isDossierDictionaryEntry())) { - - var dossier = dossierPersistenceService.findByDossierId(dossierId); - - var typeId = buildTypeId(redactionLogEntry, resizeRedaction, dossier); - var newValue = resizeRedaction.getValue(); - var oldValue = redactionLogEntry.getValue(); - var dictionaryEntryType = getDictionaryEntryType(redactionLogEntry); - - boolean isShrinking = oldValue != null && oldValue.length() > newValue.length(); - - Set typeIdsOfModifiedDictionaries = new HashSet<>(); - - if (isShrinking) { - log.info("Remove old value '{}' from dictionary", oldValue); - removeFromDictionary(typeId, oldValue, dossierId, fileId, dictionaryEntryType); - typeIdsOfModifiedDictionaries.add(typeId); - - if (resizeRedaction.isAddToAllDossiers() && redactionLogEntry.isDictionaryEntry()) { - String dossierTemplateId = dossier.getDossierTemplateId(); - var dossiersOfThisDossierTemplate = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); - var type = redactionLogEntry.getType(); - dossiersOfThisDossierTemplate.forEach(dossierEntity -> { - var typeIdOfDossierEntity = toTypeId(type, dossierTemplateId, dossierEntity.getId()); - removeFromDictionary(typeIdOfDossierEntity, oldValue, dossierId, fileId, dictionaryEntryType); - typeIdsOfModifiedDictionaries.add(typeIdOfDossierEntity); - }); - } - - } - - log.info("Add new value '{}' to dictionary", newValue); - addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType); - typeIdsOfModifiedDictionaries.add(typeId); - - resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(), - resizeRedaction.getId().getAnnotationId(), - resizeRedaction.getStatus(), - typeIdsOfModifiedDictionaries); - } - } - - - private String buildTypeId(RedactionLogEntry redactionLogEntry, ManualResizeRedactionEntity resizeRedaction, DossierEntity dossier) { - - if (resizeRedaction.isAddToAllDossiers()) { - return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId()); - } else { - return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()); - } - - } - - - private DictionaryEntryType getDictionaryEntryType(RedactionLogEntry redactionLogEntry) { - - if (redactionLogEntry.isRecommendation() && redactionLogEntry.isFalsePositive()) { - return DictionaryEntryType.FALSE_RECOMMENDATION; - } else if (redactionLogEntry.isFalsePositive()) { - return DictionaryEntryType.FALSE_POSITIVE; - } else { - return DictionaryEntryType.ENTRY; - } - } - - public ManualRedactions getManualRedactions(String fileId) { return manualRedactionProviderService.getManualRedactions(fileId); @@ -740,13 +404,4 @@ public class ManualRedactionService { } } - - public Comments getComments(String fileId) { - - return new Comments(commentPersistenceService.findCommentsByFileID(fileId, false) - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> MagicConverter.convert(entry.getValue(), Comment.class)))); - } - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionUndoService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionUndoService.java index 6eac5cba6..67e23bd93 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionUndoService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionUndoService.java @@ -4,7 +4,9 @@ import static com.iqser.red.service.persistence.service.v1.api.external.resource import static com.iqser.red.service.persistence.service.v1.api.external.resource.ManualRedactionResource.DOSSIER_ID; import static com.iqser.red.service.persistence.service.v1.api.external.resource.ManualRedactionResource.FILE_ID; +import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -13,11 +15,25 @@ import java.util.Set; import java.util.stream.Collectors; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -26,24 +42,41 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ManualRedactionWrapperModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualRedactionUndoService { - private final ManualRedactionService manualRedactionService; - private final AuditPersistenceService auditPersistenceService; - private final FileStatusService fileStatusService; + ManualRedactionProviderService manualRedactionProviderService; + AuditPersistenceService auditPersistenceService; + FileStatusService fileStatusService; + AnalysisFlagsCalculationService analysisFlagsCalculationService; + ImageRecategorizationPersistenceService recategorizationPersistenceService; + DossierPersistenceService dossierPersistenceService; + AddRedactionPersistenceService addRedactionPersistenceService; + RemoveRedactionPersistenceService removeRedactionPersistenceService; + ForceRedactionPersistenceService forceRedactionPersistenceService; + LegalBasisChangePersistenceService legalBasisChangePersistenceService; + ResizeRedactionPersistenceService resizeRedactionPersistenceService; + RedactionLogService redactionLogService; + ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler; + @Transactional public void undo(String dossierId, String fileId, Set annotationIds) { // undo the latest manual redaction for each annotationId - ManualRedactions manualRedactions = manualRedactionService.getManualRedactions(fileId); + ManualRedactions manualRedactions = getManualRedactions(fileId); Map manualRedactionWrappers = getLatestManualRedactionsForAnnotationIds(manualRedactions, annotationIds); @@ -60,6 +93,13 @@ public class ManualRedactionUndoService { reprocess(dossierId, fileId); } + + private ManualRedactions getManualRedactions(String fileId) { + + return manualRedactionProviderService.getManualRedactions(fileId); + } + + private void reprocess(String dossierId, String fileId) { fileStatusService.setStatusReprocessForManual(dossierId, fileId, true); @@ -74,7 +114,7 @@ public class ManualRedactionUndoService { .map(ManualRedactionWrapperModel::getId) .collect(Collectors.toList()); if (!manualResizeRedactions.isEmpty()) { - manualRedactionService.deleteResizeRedaction(dossierId, fileId, manualResizeRedactions); + deleteResizeRedaction(dossierId, fileId, manualResizeRedactions); manualResizeRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -86,6 +126,16 @@ public class ManualRedactionUndoService { } + private void deleteResizeRedaction(String dossierId, String fileId, List annotationIds) { + + OffsetDateTime now = OffsetDateTime.now(); + for (var annotationId : annotationIds) { + resizeRedactionPersistenceService.softDelete(fileId, annotationId, now); + } + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + } + + private void undoLegalBasisChange(String dossierId, String fileId, Map manualRedactionWrappers) { List manualLegalBasisChanges = manualRedactionWrappers.values() @@ -95,7 +145,7 @@ public class ManualRedactionUndoService { .collect(Collectors.toList()); if (!manualLegalBasisChanges.isEmpty()) { - manualRedactionService.deleteLegalBasisChange(dossierId, fileId, manualLegalBasisChanges); + deleteLegalBasisChange(dossierId, fileId, manualLegalBasisChanges); manualLegalBasisChanges.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -107,6 +157,16 @@ public class ManualRedactionUndoService { } + private void deleteLegalBasisChange(String dossierId, String fileId, List annotationIds) { + + for (var annotationId : annotationIds) { + legalBasisChangePersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + } + + private void undoRecategorization(String dossierId, String fileId, Map manualRedactionWrappers) { List manualImageRecategorizations = manualRedactionWrappers.values() @@ -116,7 +176,7 @@ public class ManualRedactionUndoService { .collect(Collectors.toList()); if (!manualImageRecategorizations.isEmpty()) { - manualRedactionService.deleteRecategorization(dossierId, fileId, manualImageRecategorizations); + deleteRecategorization(dossierId, fileId, manualImageRecategorizations); manualImageRecategorizations.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -128,6 +188,22 @@ public class ManualRedactionUndoService { } + private void deleteRecategorization(String dossierId, String fileId, List annotationIds) { + + dossierPersistenceService.getAndValidateDossier(dossierId); + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); + + for (var annotationId : annotationIds) { + ManualImageRecategorizationEntity recategorizationEntity = recategorizationPersistenceService.findRecategorization(fileId, annotationId); + String originalValue = getRedactionLogEntry(redactionLog, annotationId).getValue(); + manualRedactionDictionaryUpdateHandler.revertRemoveFromDictionary(originalValue, dossierId, fileId, recategorizationEntity.getTypeIdsOfDictionariesWithDelete()); + manualRedactionDictionaryUpdateHandler.revertAddToDictionary(originalValue, DictionaryEntryType.ENTRY, fileId, dossierId, recategorizationEntity.getTypeIdsOfDictionariesWithAdd()); + recategorizationPersistenceService.updateStatus(fileId, annotationId, AnnotationStatus.APPROVED, false, Collections.emptySet(), Collections.emptySet()); + recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); + } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + } + + private void undoForceRedactions(String dossierId, String fileId, Map manualRedactionWrappers) { List manualForceRedactions = manualRedactionWrappers.values() @@ -137,7 +213,7 @@ public class ManualRedactionUndoService { .collect(Collectors.toList()); if (!manualForceRedactions.isEmpty()) { - manualRedactionService.deleteForceRedaction(dossierId, fileId, manualForceRedactions); + deleteForceRedaction(dossierId, fileId, manualForceRedactions); manualForceRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -149,6 +225,18 @@ public class ManualRedactionUndoService { } + private void deleteForceRedaction(String dossierId, String fileId, List annotationIds) { + + var now = OffsetDateTime.now(); + + for (var annotationId : annotationIds) { + forceRedactionPersistenceService.softDelete(fileId, annotationId, now); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + } + + private void undoIdRemovals(String dossierId, String fileId, Map manualRedactionWrappers) { List idRemovals = manualRedactionWrappers.values() @@ -157,7 +245,7 @@ public class ManualRedactionUndoService { .map(ManualRedactionWrapperModel::getId) .collect(Collectors.toList()); if (!idRemovals.isEmpty()) { - manualRedactionService.deleteRemoveRedaction(dossierId, fileId, idRemovals); + deleteRemoveRedaction(dossierId, fileId, idRemovals); idRemovals.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -169,6 +257,24 @@ public class ManualRedactionUndoService { } + private void deleteRemoveRedaction(String dossierId, String fileId, List annotationIds) { + + dossierPersistenceService.getAndValidateDossier(dossierId); + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId); + + for (String annotationId : annotationIds) { + + IdRemovalEntity removeRedaction = removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId); + String originalValue = getRedactionLogEntry(redactionLog, annotationId).getValue(); + boolean dictionaryChanged = manualRedactionDictionaryUpdateHandler.revertRemoveFromDictionary(originalValue, dossierId, fileId, removeRedaction); + removeRedactionPersistenceService.updateStatus(fileId, annotationId, removeRedaction.getStatus(), dictionaryChanged, Collections.emptySet()); + removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + } + + private void undoManualRedactionEntries(String dossierId, String fileId, Map manualRedactionWrappers) { List manualRedactionEntries = manualRedactionWrappers.values() @@ -177,7 +283,7 @@ public class ManualRedactionUndoService { .map(ManualRedactionWrapperModel::getId) .collect(Collectors.toList()); if (!manualRedactionEntries.isEmpty()) { - manualRedactionService.deleteAddRedaction(dossierId, fileId, manualRedactionEntries); + deleteAddRedaction(dossierId, fileId, manualRedactionEntries); manualRedactionEntries.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -189,6 +295,33 @@ public class ManualRedactionUndoService { } + private void deleteAddRedaction(String dossierId, String fileId, List annotationIds) { + + var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); + + for (String annotationId : annotationIds) { + + ManualRedactionEntryEntity addRedaction = addRedactionPersistenceService.findAddRedaction(fileId, annotationId); + + boolean dictionaryChanged = manualRedactionDictionaryUpdateHandler.revertAddToDictionary(fileId, dossier.getId(), addRedaction); + addRedactionPersistenceService.updateStatus(fileId, annotationId, AnnotationStatus.APPROVED, dictionaryChanged, null); + addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + } + + + private RedactionLogEntry getRedactionLogEntry(RedactionLog redactionLog, String annotationId) { + + return redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getId().equals(annotationId)) + .findFirst() + .orElseThrow(() -> new NotFoundException("Annotation does not exist in redaction log.")); + } + + private Map getLatestManualRedactionsForAnnotationIds(ManualRedactions manualRedactions, Set annotationIds) { Map result = new HashMap<>(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java index a2151494c..e3738e752 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java @@ -33,6 +33,7 @@ public class ImageRecategorizationPersistenceService { manualImageRecategorization.setId(new AnnotationEntityId(recategorizationRequest.getAnnotationId(), fileId)); BeanUtils.copyProperties(recategorizationRequest, manualImageRecategorization); manualImageRecategorization.setRequestDate(OffsetDateTime.now()); + manualImageRecategorization.setTypeId(recategorizationRequest.getDossierTemplateTypeId()); imageRecategorizationRepository.saveAndFlush(manualImageRecategorization); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualImageRecategorizationMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualImageRecategorizationMapper.java index 567236630..6cac92b0d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualImageRecategorizationMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ManualImageRecategorizationMapper.java @@ -8,9 +8,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations public class ManualImageRecategorizationMapper implements BiConsumer { @Override - public void accept(ManualImageRecategorizationEntity manualRedactionEntryEntity, ManualImageRecategorization manualRedactionEntry) { + public void accept(ManualImageRecategorizationEntity manualImageRecategorizationEntity, ManualImageRecategorization manualRedactionEntry) { - manualRedactionEntry.setType(manualRedactionEntryEntity.getTypeId().substring(0, manualRedactionEntryEntity.getTypeId().indexOf(":"))); + manualRedactionEntry.setType(manualImageRecategorizationEntity.getTypeId().substring(0, manualImageRecategorizationEntity.getTypeId().indexOf(":"))); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java index 44911a350..c245080bf 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java @@ -2,8 +2,6 @@ package com.iqser.red.service.peristence.v1.server.integration.service; import static org.assertj.core.api.Assertions.assertThat; -import java.awt.Color; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -11,9 +9,7 @@ import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryC import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; @Service public class TypeProvider { @@ -21,6 +17,7 @@ public class TypeProvider { @Autowired private DictionaryClient dictionaryClient; + public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate) { return testAndProvideType(dossierTemplate, null, "test", false); @@ -32,9 +29,13 @@ public class TypeProvider { return testAndProvideType(dossierTemplate, dossier, typeName, false); } + public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate, Dossier dossier, String typeName, boolean dossierDictionaryOnly) { + return testAndProvideType(dossierTemplate, dossier, typeName, dossierDictionaryOnly, 100); } + + public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate, Dossier dossier, String typeName, boolean dossierDictionaryOnly, int rank) { var type = new CreateTypeValue(); @@ -53,9 +54,9 @@ public class TypeProvider { type.setDossierDictionaryOnly(dossierDictionaryOnly); dictionaryClient.addType(type); - var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getDossierTemplateId(),dossier != null ? dossier.getId() : null,false); + var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getDossierTemplateId(), dossier != null ? dossier.getId() : null, false); - var foundType =allTypes.getTypes().stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findAny(); + var foundType = allTypes.getTypes().stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findAny(); assertThat(foundType).isPresent(); return foundType.get(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index fbf87c741..fa626540e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -1,32 +1,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import com.google.common.collect.Sets; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; -import com.iqser.red.service.peristence.v1.server.integration.client.*; -import com.iqser.red.service.peristence.v1.server.integration.service.*; -import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.*; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel; - -import feign.FeignException; -import lombok.SneakyThrows; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.mock.web.MockMultipartFile; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; @@ -36,7 +11,58 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.assertj.core.api.Assertions.assertThat; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.mock.web.MockMultipartFile; + +import com.google.common.collect.Sets; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; +import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileManagementClient; +import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; +import com.iqser.red.service.peristence.v1.server.integration.client.ReanalysisClient; +import com.iqser.red.service.peristence.v1.server.integration.client.RedactionLogClient; +import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient; +import com.iqser.red.service.peristence.v1.server.integration.client.ViewedPagesClient; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider; +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusionRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ViewedPagesRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; + +import feign.FeignException; +import lombok.SneakyThrows; public class FileTest extends AbstractPersistenceServerServiceTest { @@ -103,7 +129,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - fileManagementClient.deleteFile(dossier.getId(), file.getId()); var nrOfFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()).size(); @@ -330,31 +355,37 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual"); + var annotationId = "imagine_this_makes_sense"; + RedactionLog redactionLog = new RedactionLog(); + RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder().type(type.getType()).id(annotationId).build(); + redactionLog.getRedactionLogEntry().add(redactionLogEntry); + + when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog); + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); - var addRedaction = manualRedactionClient.addRedactionBulk(dossierId, - fileId, - Set.of(AddRedactionRequestModel.builder() - .addToDictionary(true) - .type(type.getType()) - .addToAllDossiers(true) - .reason("1") - .value("test") - .legalBasis("1") - .dictionaryEntryType(DictionaryEntryType.ENTRY) - .build())).iterator().next(); + var addRedactionRequest = AddRedactionRequestModel.builder() + .addToDictionary(true) + .type(type.getType()) + .addToAllDossiers(true) + .reason("1") + .value("test") + .legalBasis("1") + .build(); + manualRedactionClient.addRedactionBulk(dossierId, fileId, Set.of(addRedactionRequest)); + manualRedactionClient.removeRedactionBulk(dossierId, fileId, - Set.of(RemoveRedactionRequestModel.builder().annotationId(addRedaction.getAnnotationId()).comment("comment").removeFromDictionary(false).build())); + Set.of(RemoveRedactionRequestModel.builder().annotationId(annotationId).comment("comment").removeFromDictionary(false).build())); manualRedactionClient.forceRedactionBulk(dossierId, fileId, Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation").comment("comment").legalBasis("1").build())); - manualRedactionClient.legalBasisChangeBulk(dossierId, + manualRedactionClient.legalBasisChangeBulk(dossierId, fileId, Set.of(LegalBasisChangeRequestModel.builder().annotationId("legalBasisChangeAnnotation").comment("comment").legalBasis("1").build())); manualRedactionClient.recategorizeBulk(dossierId, fileId, - Set.of(RecategorizationRequestModel.builder().annotationId("imageRecategorizationAnnotation").comment("comment").type("new-type").build())); + Set.of(RecategorizationRequestModel.builder().annotationId(annotationId).comment("comment").type("new-type").build())); var loadedFile = fileClient.getFileStatus(dossierId, fileId); @@ -409,12 +440,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest { .reason("1") .value("test") .legalBasis("1") - .dictionaryEntryType(DictionaryEntryType.ENTRY) .build())).iterator().next(); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - reanalysisClient.toggleExclusion(dossier.getId(), file.getId(), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.isExcluded()).isTrue(); @@ -438,7 +467,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier, fileName); String fileId = file.getId(); - var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual"); assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); @@ -515,6 +543,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { } + @Test public void testFile6034SetUnderReview() { @@ -542,17 +571,15 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(actualMessage).contains(expectedMessage); - exception = Assertions.assertThrows(FeignException.BadRequest.class, () -> { fileClient.setStatusUnderReview(dossier.getId(), file.getId(), user2); }); expectedMessage = "User must be dossier member"; - actualMessage = exception.getMessage(); + actualMessage = exception.getMessage(); assertThat(actualMessage).contains(expectedMessage); - fileClient.setStatusUnderReview(dossier.getId(), file.getId(), altUserId); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); @@ -561,6 +588,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastApprover()).isNull(); } + @Test @SneakyThrows public void testUploadFileIsInOcrProcessingWithOcrByDefaultFlagTrue() { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 73a74774b..ab73a294a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -13,10 +13,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Redacti import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.Dictionary; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ResizeRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -238,10 +240,10 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), false, dossier.getDossierId(), DictionaryEntryType.ENTRY); - var dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + Dictionary dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker", "Darth Vader"); - var dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + Dictionary dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker"); var redactionLog = new RedactionLog(1, @@ -258,7 +260,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), - Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).removeFromAllDossiers(true).build())).get(0); + Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).removeFromAllDossiers(true).build())); dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Darth Vader"); @@ -948,4 +950,73 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertThat(mergedDictForTypeDosTempDictInDossier2.getEntries().get(0)).isEqualTo("test redaction in dossier template"); } + @Test + public void testManualRecategorizeAndUndo() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate); + var type2 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test2", false, 66); + + var entries = new ArrayList(); + + var lukeSkywalker = "Luke Skywalker"; + var darthVader = "Darth Vader"; + entries.add(lukeSkywalker); + entries.add(darthVader); + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY); + + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of(lukeSkywalker), false, dossier.getDossierId(), DictionaryEntryType.ENTRY); + + Dictionary dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker, darthVader); + Dictionary dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker); + + Dictionary dossierTemplateDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary2.getEntries()).isEmpty(); + Dictionary dossierDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary2.getEntries()).isEmpty(); + + var annotationId = "AnnotationId"; + var redactionLog = new RedactionLog(1, + 1, + List.of(RedactionLogEntry.builder().id(annotationId).type(type.getType()).value(lukeSkywalker).isDictionaryEntry(true).build()), + null, + 0, + 0, + 0, + 0); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog); + + when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog); + + manualRedactionClient.recategorizeBulk(dossier.getId(), + file.getId(), + Set.of(RecategorizationRequestModel.builder().type(type2.getType()).annotationId(annotationId).addToDictionary(true).addToAllDossiers(true).build())); + + dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(darthVader); + dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary.getEntries()).isEmpty(); + + dossierTemplateDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary2.getEntries()).containsExactlyInAnyOrder(lukeSkywalker); + dossierDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary2.getEntries()).containsExactlyInAnyOrder(lukeSkywalker); + + manualRedactionClient.undo(dossier.getDossierId(), file.getFileId(), Set.of(annotationId)); + + dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker, darthVader); + dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder(lukeSkywalker); + + dossierTemplateDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary2.getEntries()).isEmpty(); + dossierDictionary2 = dictionaryClient.getDictionaryForType(type2.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary2.getEntries()).isEmpty(); + } } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithAddToDictionary.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithAddToDictionary.java index 200276fa5..d06ef8539 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithAddToDictionary.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithAddToDictionary.java @@ -4,12 +4,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp public interface ManualRequestWithAddToDictionary extends BaseManualRequest { - String getValue(); String getDossierId(); + + boolean isAddToDictionary(); + + boolean isAddToAllDossiers(); + + boolean isForceAddToDictionary(); + + String getDossierTemplateTypeId(); + + DictionaryEntryType getDictionaryEntryType(); } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java index 269047687..1a7e371a3 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java @@ -23,7 +23,6 @@ public class RecategorizationRequest implements ManualRequestWithAddToDictionary String dossierId; String comment; int page; - String value; boolean addToDictionary; boolean addToAllDossiers; private DictionaryEntryType dictionaryEntryType;