From 7beee690f50d320c4909fadbf1c44cd282806ae3 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 2 Jun 2023 10:40:10 +0200 Subject: [PATCH] RED-6798, RED-6751, RED-6545: Enabled to add delete entries, changed permissions for manual redactions, implemented logic for addToAllDossiers --- .../controller/ManualRedactionController.java | 35 ++- .../entity/annotations/IdRemovalEntity.java | 12 + .../ManualRedactionEntryEntity.java | 11 + .../service/DictionaryManagementService.java | 27 +- .../service/ManualRedactionService.java | 90 +++++-- .../persistence/EntryPersistenceService.java | 29 ++- .../AddRedactionPersistenceService.java | 17 +- .../RemoveRedactionPersistenceService.java | 15 +- .../RemoveRedactionRepository.java | 6 - .../dictionaryentry/EntryRepository.java | 3 + .../db/changelog/db.changelog-tenant.yaml | 4 + ...ll-dossier-to-manual-remove-redaction.yaml | 46 ++++ ...o-all-dossier-to-manual-add-redaction.yaml | 46 ++++ .../client/InternalDictionaryClient.java | 10 + .../v1/server/integration/tests/FileTest.java | 6 +- .../tests/ManualRedactionTest.java | 233 +++++++++++++++++- .../annotations/AddRedactionRequest.java | 5 +- .../annotations/RemoveRedactionRequest.java | 1 + .../annotations/entitymapped/IdRemoval.java | 5 +- .../model/manual/AddRedactionRequest.java | 2 + .../model/manual/RemoveRedactionRequest.java | 1 + 21 files changed, 543 insertions(+), 61 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/103-added-remove-from-all-dossier-to-manual-remove-redaction.yaml create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-added-add-to-all-dossier-to-manual-add-redaction.yaml create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/client/InternalDictionaryClient.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java index bf43ff689..792a507fd 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java @@ -1,6 +1,12 @@ package com.iqser.red.persistence.service.v1.external.api.impl.controller; -import static com.iqser.red.keycloak.commons.roles.ActionRoles.*; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.ADD_COMMENT; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.DELETE_COMMENT; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.DELETE_MANUAL_REDACTION; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.DO_MANUAL_REDACTION; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.PROCESS_MANUAL_REDACTION_REQUEST; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.READ_MANUAL_REDACTIONS; +import static com.iqser.red.keycloak.commons.roles.ActionRoles.REQUEST_MANUAL_REDACTION; import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; import java.util.ArrayList; @@ -93,7 +99,7 @@ public class ManualRedactionController implements ManualRedactionResource { var addRedactionRequestBuilder = com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest.builder() .user(KeycloakSecurity.getUserId()) - .typeId(toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId(), addRedactionRequest.isAddToDossierDictionary() ? dossierId : null)) + .dossierTemplateTypeId(toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId())) .value(addRedactionRequest.getValue()) .reason(addRedactionRequest.getReason()) .legalBasis(addRedactionRequest.getLegalBasis()) @@ -102,7 +108,7 @@ public class ManualRedactionController implements ManualRedactionResource { .section(addRedactionRequest.getSection()) .positions(addRedactionRequest.getPositions()) .comment(addRedactionRequest.getComment() != null ? addRedactionRequest.getComment().getText() : null) - .addToDossierDictionary(addRedactionRequest.isAddToDossierDictionary()) + .addToAllDossiers(addRedactionRequest.isAddToAllDossiers()) .forceAddToDictionary(addRedactionRequest.isForceAddToDictionary()) .rectangle(addRedactionRequest.isRectangle()) .dictionaryEntryType(addRedactionRequest.getDictionaryEntryType()) @@ -500,7 +506,7 @@ public class ManualRedactionController implements ManualRedactionResource { public void undo(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set annotationIds) { accessControlService.verifyFileIsNotApproved(dossierId, fileId); - accessControlService.verifyUserIsReviewer(dossierId, fileId); + accessControlService.verifyUserIsApprover(dossierId); ManualRedactions manualRedactions = manualRedactionService.getManualRedactions(fileId); @@ -873,8 +879,13 @@ public class ManualRedactionController implements ManualRedactionResource { @RequestBody Set addRedactionRequests) { var dossier = dossierManagementService.getDossierById(dossierId, false, false); + accessControlService.verifyFileIsNotApproved(dossierId, fileId); - accessControlService.verifyUserIsApprover(dossierId); + if(addRedactionRequests.stream().anyMatch(AddRedactionRequest::isAddToAllDossiers)){ + accessControlService.verifyUserIsApprover(dossierId); + } else { + accessControlService.verifyUserIsMemberOrApprover(dossierId); + } List requests = new ArrayList<>(); @@ -884,17 +895,18 @@ public class ManualRedactionController implements ManualRedactionResource { .value(addRedactionRequest.getValue()) .legalBasis(addRedactionRequest.getLegalBasis()) .user(KeycloakSecurity.getUserId()) - .typeId(toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId(), addRedactionRequest.isAddToDossierDictionary() ? dossierId : null)) + .dossierTemplateTypeId(toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId())) .reason(addRedactionRequest.getReason()) .addToDictionary(addRedactionRequest.isAddToDictionary()) .status(AnnotationStatus.APPROVED) .comment(addRedactionRequest.getComment() != null ? addRedactionRequest.getComment().getText() : null) .section(addRedactionRequest.getSection()) .rectangle(addRedactionRequest.isRectangle()) - .addToDossierDictionary(addRedactionRequest.isAddToDossierDictionary()) + .addToAllDossiers(addRedactionRequest.isAddToAllDossiers()) .forceAddToDictionary(addRedactionRequest.isForceAddToDictionary()) .positions(addRedactionRequest.getPositions()) .sourceId(addRedactionRequest.getSourceId()) + .dossierId(dossierId) .dictionaryEntryType(addRedactionRequest.getDictionaryEntryType()); requests.add(addRedactionRequestBuilder.build()); @@ -928,7 +940,11 @@ public class ManualRedactionController implements ManualRedactionResource { @RequestBody Set removeRedactionRequests) { accessControlService.verifyFileIsNotApproved(dossierId, fileId); - accessControlService.verifyUserIsApprover(dossierId); + if(removeRedactionRequests.stream().anyMatch(RemoveRedactionRequest::isRemoveFromAllDossiers)){ + accessControlService.verifyUserIsApprover(dossierId); + } else { + accessControlService.verifyUserIsMemberOrApprover(dossierId); + } List requests = new ArrayList<>(); @@ -937,7 +953,8 @@ public class ManualRedactionController implements ManualRedactionResource { .annotationId(removeRedactionRequest.getAnnotationId()) .user(KeycloakSecurity.getUserId()) .status(AnnotationStatus.APPROVED) - .removeFromDictionary(removeRedactionRequest.isRemoveFromDictionary()); + .removeFromDictionary(removeRedactionRequest.isRemoveFromDictionary()) + .removeFromAllDossiers(removeRedactionRequest.isRemoveFromAllDossiers()); if (removeRedactionRequest.getComment() != null) { removeRedactionRequestBuilder.comment(removeRedactionRequest.getComment()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java index a9b33251b..a51b61868 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java @@ -1,8 +1,11 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotations; import java.time.OffsetDateTime; +import java.util.HashSet; +import java.util.Set; import javax.persistence.Column; +import javax.persistence.ElementCollection; import javax.persistence.EmbeddedId; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -10,6 +13,9 @@ import javax.persistence.Enumerated; import javax.persistence.ManyToOne; import javax.persistence.Table; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; @@ -36,6 +42,8 @@ public class IdRemovalEntity implements IBaseAnnotation { @Column private boolean removeFromDictionary; @Column + private boolean removeFromAllDossiers; + @Column private OffsetDateTime requestDate; @Column private OffsetDateTime processedDate; @@ -47,4 +55,8 @@ public class IdRemovalEntity implements IBaseAnnotation { @ManyToOne private FileEntity fileStatus; + @ElementCollection + @Fetch(value = FetchMode.SUBSELECT) + private Set typeIdsOfModifiedDictionaries = new HashSet<>(); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java index 815170ddd..20ba0f1d3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java @@ -2,7 +2,9 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.persistence.Column; import javax.persistence.ElementCollection; @@ -56,7 +58,12 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation { @Column private boolean addToDictionary; @Column + private boolean addToAllDossiers; + + @Column + @Deprecated private boolean addToDossierDictionary; + @Column @Enumerated(EnumType.STRING) private DictionaryEntryType dictionaryEntryType; @@ -82,4 +89,8 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation { @Column private String sourceId; + @ElementCollection + @Fetch(value = FetchMode.SUBSELECT) + private Set typeIdsOfModifiedDictionaries = new HashSet<>(); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java index 3b2296f39..9aae5f37b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java @@ -26,6 +26,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConv import com.iqser.red.service.persistence.management.v1.processor.utils.TextNormalizationUtilities; import com.iqser.red.service.persistence.management.v1.processor.utils.TypeMapper; import com.iqser.red.service.persistence.management.v1.processor.validation.DictionaryValidator; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; @@ -43,6 +44,11 @@ public class DictionaryManagementService { private final StopwordService stopwordService; + public List getAllEntriesInDossierTemplate(String dossierTemplateTypeId, String value){ + return MagicConverter.convert(entryPersistenceService.getAllEntriesInDossierTemplate(dossierTemplateTypeId, value),DictionaryEntry.class); + } + + @Transactional public Type addType(Type typeRequest) { @@ -273,14 +279,29 @@ public class DictionaryManagementService { var currentVersion = getCurrentVersion(typeResult); + List allExistingEntiesFromType = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).toList(); + if (typeResult.isCaseInsensitive()) { - List existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).toList(); + + // Set existing entries to deleted. + var caseInsensitiveExistingEntries = allExistingEntiesFromType.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toSet()); entryPersistenceService.deleteEntries(typeId, - existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toSet()), + caseInsensitiveExistingEntries, currentVersion + 1, dictionaryEntryType); + + // Create new deleted entries for not existing. + var caseInsensitiveNonExistingEntries = entries.stream().filter(e -> caseInsensitiveExistingEntries.stream().noneMatch(e::equalsIgnoreCase)).collect(toSet()); + entryPersistenceService.addDeleteEntries(typeId, caseInsensitiveNonExistingEntries, currentVersion + 1, dictionaryEntryType); } else { - entryPersistenceService.deleteEntries(typeId, new HashSet<>(entries), currentVersion + 1, dictionaryEntryType); + + // Set existing entries to deleted. + var caseSensitiveExistingEntries = allExistingEntiesFromType.stream().filter(e -> entries.stream().anyMatch(e::equals)).collect(toSet()); + entryPersistenceService.deleteEntries(typeId, new HashSet<>(caseSensitiveExistingEntries), currentVersion + 1, dictionaryEntryType); + + // Create new deleted entries for not existing. + var nonExistingEntries = entries.stream().filter(e -> caseSensitiveExistingEntries.stream().noneMatch(e::equals)).collect(toSet()); + entryPersistenceService.addDeleteEntries(typeId, nonExistingEntries, currentVersion + 1, dictionaryEntryType); } dictionaryPersistenceService.incrementVersion(typeId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java index 39097647a..21496594f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ManualRedactionService.java @@ -12,9 +12,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -26,9 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; @@ -109,7 +105,7 @@ public class ManualRedactionService { var actionPerformed = false; for (var addRedactionRequest : addRedactionRequests) { - if (addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary()) { + if (addRedactionRequest.isAddToDictionary()) { validateDictionary(addRedactionRequest); } @@ -124,11 +120,11 @@ public class ManualRedactionService { var addedToDictionary = handleAddToDictionary(fileId, annotationId, - addRedactionRequest.getTypeId(), + addRedactionRequest.getDossierTemplateTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), - addRedactionRequest.isAddToDossierDictionary(), + addRedactionRequest.isAddToAllDossiers(), false, dossierId, addRedactionRequest.getDictionaryEntryType()); @@ -147,7 +143,7 @@ public class ManualRedactionService { var manualTextRedactions = MagicConverter.convert(response.stream() .map(r -> getAddRedaction(fileId, r.getAnnotationId())) - .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) + .filter(m -> !m.isAddToDictionary() && !m.isRectangle()) .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); if (!manualTextRedactions.isEmpty()) { @@ -167,9 +163,9 @@ public class ManualRedactionService { if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) { throw new ConflictException("The entry you are trying to add is a stopword"); } - dictionaryPersistenceService.getType(addRedactionRequest.getTypeId()); + dictionaryPersistenceService.getType(addRedactionRequest.getDossierTemplateTypeId()); } catch (NotFoundException e) { - throw new BadRequestException("Invalid type: " + addRedactionRequest.getTypeId()); + throw new BadRequestException("Invalid type: " + addRedactionRequest.getDossierTemplateTypeId()); } } @@ -195,27 +191,42 @@ public class ManualRedactionService { private boolean handleAddToDictionary(String fileId, String annotationId, - String typeId, + String dossierTemplateTypeId, String value, AnnotationStatus status, boolean addToDictionary, - boolean addToDossierDictionary, + boolean addToAllDossiers, boolean revert, String dossierId, DictionaryEntryType dictionaryEntryType) { if (status == AnnotationStatus.APPROVED) { - addRedactionPersistenceService.updateStatus(fileId, annotationId, status, addToDictionary, addToDossierDictionary); - if (addToDictionary || addToDossierDictionary) { + if (addToDictionary) { + Set typeIdsOfModifiedDictionaries = new HashSet<>(); if (revert) { - removeFromDictionary(typeId, value, dossierId, fileId, dictionaryEntryType); + var addRedactionToRevert = addRedactionPersistenceService.findAddRedaction(fileId, annotationId); + addRedactionToRevert.getTypeIdsOfModifiedDictionaries().forEach(typeId -> { + removeFromDictionary(typeId, value, dossierId, fileId, dictionaryEntryType); + }); } else { - addToDictionary(typeId, value, dossierId, fileId, dictionaryEntryType); + if (addToAllDossiers) { + var dictionaryEntriesToUnDelete = dictionaryService.getAllEntriesInDossierTemplate(dossierTemplateTypeId, value); + dictionaryEntriesToUnDelete.forEach(entry -> { + typeIdsOfModifiedDictionaries.add(entry.getTypeId()); + addToDictionary(entry.getTypeId(), value, dossierId, fileId, dictionaryEntryType); + }); + addToDictionary(dossierTemplateTypeId, value, dossierId, fileId, dictionaryEntryType); + typeIdsOfModifiedDictionaries.add(dossierTemplateTypeId); + } else { + addToDictionary(dossierTemplateTypeId + ":" + dossierId, value, dossierId, fileId, dictionaryEntryType); + typeIdsOfModifiedDictionaries.add(dossierTemplateTypeId + ":" + dossierId); + } + addRedactionPersistenceService.updateStatus(fileId, annotationId, status, true, typeIdsOfModifiedDictionaries); } return true; } - + addRedactionPersistenceService.updateStatus(fileId, annotationId, status, false, null); } return false; } @@ -226,6 +237,7 @@ public class ManualRedactionService { fileStatusService.setStatusReprocessForManual(dossierId, fileId, true); } + @Transactional public ManualRedactionEntryEntity getAddRedaction(String fileId, String annotationId) { @@ -237,9 +249,7 @@ public class ManualRedactionService { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING); - var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId) - .manualRedactions(manualRedactions) - .build(); + var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build(); try { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> { @@ -274,6 +284,7 @@ public class ManualRedactionService { } } + @Transactional public List addRemoveRedaction(String dossierId, String fileId, List removeRedactionRequests) { @@ -321,6 +332,7 @@ public class ManualRedactionService { removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), + removeRedactionRequest.isRemoveFromAllDossiers(), false); if (!removedFromDictionary && idRemoval.isApproved()) { @@ -355,10 +367,13 @@ public class ManualRedactionService { String annotationId, AnnotationStatus status, boolean removeFromDictionary, + boolean removeFromAllDossiers, boolean revert) { if (status == AnnotationStatus.APPROVED) { - removeRedactionPersistenceService.updateStatus(fileId, annotationId, status, removeFromDictionary); + + Set typeIdsOfModifiedDictionaries = new HashSet<>(); + if (removeFromDictionary) { Optional redactionLogEntryOptional = redactionLog.getRedactionLogEntry() @@ -373,15 +388,35 @@ public class ManualRedactionService { var redactionLogEntry = redactionLogEntryOptional.get(); if (revert) { - addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY); + var idRemovalEntity = removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId); + idRemovalEntity.getTypeIdsOfModifiedDictionaries().forEach(changedTypeId -> { + addToDictionary(changedTypeId, redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY); + }); } else { - removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY); + if (removeFromAllDossiers) { + var dictionaryEntriesToRemove = dictionaryService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(),dossier.getDossierTemplateId()), + redactionLogEntry.getValue()); + dictionaryEntriesToRemove.forEach(entry -> { + typeIdsOfModifiedDictionaries.add(entry.getTypeId()); + removeFromDictionary(entry.getTypeId(), entry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY); + }); + } else { + typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId())); + removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()), + redactionLogEntry.getValue(), + dossier.getId(), + fileId, + DictionaryEntryType.ENTRY); + } // This is needed to remove resizeRedactions with addToDictionary. removeResizeRedactionsWithAddToDictionary(dossier.getDossierTemplateId(), redactionLogEntry.getValue()); } + + removeRedactionPersistenceService.updateStatus(fileId, annotationId, status, true, typeIdsOfModifiedDictionaries); return true; } + removeRedactionPersistenceService.updateStatus(fileId, annotationId, status, false, typeIdsOfModifiedDictionaries); } return false; } @@ -409,6 +444,7 @@ public class ManualRedactionService { }); } + @Transactional public List addForceRedaction(String dossierId, String fileId, List forceRedactionRequests) { @@ -437,6 +473,7 @@ public class ManualRedactionService { return response; } + @Transactional public List addLegalBasisChange(String dossierId, String fileId, List legalBasisChangeRequests) { @@ -459,6 +496,7 @@ public class ManualRedactionService { return response; } + @Transactional public List addImageRecategorization(String dossierId, String fileId, List imageRecategorizationRequests) { @@ -516,7 +554,7 @@ public class ManualRedactionService { addRedaction.getValue(), addRedaction.getStatus(), addRedaction.isAddToDictionary(), - addRedaction.isAddToDossierDictionary(), + addRedaction.isAddToAllDossiers(), true, dossier.getId(), addRedaction.getDictionaryEntryType()); @@ -548,6 +586,7 @@ public class ManualRedactionService { annotationId, removeRedaction.getStatus(), removeRedaction.isRemoveFromDictionary(), + removeRedaction.isRemoveFromAllDossiers(), true); actionPerformed = actionPerformed || removedFromDictionary; @@ -638,6 +677,7 @@ public class ManualRedactionService { analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } + @Transactional public List addResizeRedaction(String dossierId, String fileId, List resizeRedactionRequests) { @@ -711,6 +751,7 @@ public class ManualRedactionService { } } + @Transactional @SuppressWarnings("PMD") public void updateRemoveRedactionStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { @@ -822,6 +863,7 @@ public class ManualRedactionService { analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } + @Transactional public void updateImageRecategorizationStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java index 22b3b46a2..87c7afb1b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java @@ -34,6 +34,21 @@ public class EntryPersistenceService { private final JDBCWriteUtils jdbcWriteUtils; + public List getAllEntriesInDossierTemplate(String dossierTemplateTypeId, String value){ + return entryRepository.findByTypeIdContainsAndValue(dossierTemplateTypeId, value); + } + + @Transactional + public void insertDeleteEntries(String typeId, Set values, long version, DictionaryEntryType dictionaryEntryType) { + + switch (dictionaryEntryType) { + case ENTRY -> entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version); + case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version); + case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version); + } + } + + @Transactional public void deleteEntries(String typeId, Set values, long version, DictionaryEntryType dictionaryEntryType) { @@ -75,9 +90,18 @@ public class EntryPersistenceService { } } - @Transactional public void addEntries(String typeId, Set entries, long version, DictionaryEntryType dictionaryEntryType) { + addEntries(typeId, entries, version, dictionaryEntryType, false); + } + + @Transactional + public void addDeleteEntries(String typeId, Set entries, long version, DictionaryEntryType dictionaryEntryType) { + addEntries(typeId, entries, version, dictionaryEntryType, true); + } + + @Transactional + public void addEntries(String typeId, Set entries, long version, DictionaryEntryType dictionaryEntryType, boolean deleted) { var type = typeRepository.getById(typeId); @@ -92,6 +116,7 @@ public class EntryPersistenceService { entry.setVersion(version); entry.setValue(e); entry.setTypeId(type.getId()); + entry.setDeleted(deleted); return entry; }).collect(Collectors.toList()); @@ -107,6 +132,7 @@ public class EntryPersistenceService { entry.setVersion(version); entry.setValue(e); entry.setTypeId(type.getId()); + entry.setDeleted(deleted); return entry; }).collect(Collectors.toList()); @@ -121,6 +147,7 @@ public class EntryPersistenceService { entry.setVersion(version); entry.setValue(e); entry.setTypeId(type.getId()); + entry.setDeleted(deleted); return entry; }).collect(Collectors.toList()); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index 76e9ac6b6..5ec8d03de 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -38,10 +38,10 @@ public class AddRedactionPersistenceService { BeanUtils.copyProperties(addRedactionRequest, manualRedactionEntry); manualRedactionEntry.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); manualRedactionEntry.setPositions(convert(addRedactionRequest.getPositions())); - manualRedactionEntry.setTypeId(addRedactionRequest.getTypeId()); + manualRedactionEntry.setTypeId(addRedactionRequest.getDossierTemplateTypeId()); manualRedactionEntry.setDictionaryEntryType(addRedactionRequest.getDictionaryEntryType()); - if (addRedactionRequest.getStatus() == AnnotationStatus.APPROVED && !(addRedactionRequest.isAddToDictionary() || addRedactionRequest.isAddToDossierDictionary())) { + if (addRedactionRequest.getStatus() == AnnotationStatus.APPROVED && !(addRedactionRequest.isAddToDictionary())) { manualRedactionEntry.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } @@ -115,18 +115,25 @@ public class AddRedactionPersistenceService { manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, processedDate); } - @Transactional public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary, - boolean isAddOrRemoveFromDossierDictionary) { + Set typeIdsOfModifiedDictionaries) { - manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, isAddOrRemoveFromDictionary, isAddOrRemoveFromDossierDictionary); + var addRedaction = manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId)) + .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); + + addRedaction.setStatus(annotationStatus); + addRedaction.setAddToDictionary(isAddOrRemoveFromDictionary); + addRedaction.setTypeIdsOfModifiedDictionaries(typeIdsOfModifiedDictionaries); + + manualRedactionRepository.saveAndFlush(addRedaction); } + @Transactional public void approveStatusForRequestedRedactionsWithSameValue(Set fileIds, String value) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java index e71683dd7..7ce4ecde9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java @@ -88,9 +88,20 @@ public class RemoveRedactionPersistenceService { @Transactional - public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary) { + public void updateStatus(String fileId, + String annotationId, + AnnotationStatus annotationStatus, + boolean isAddOrRemoveFromDictionary, + Set typeIdsOfModifiedDictionaries) { - removeRedactionRepository.updateStatusAndRemoveFromDictionary(new AnnotationEntityId(annotationId, fileId), annotationStatus, isAddOrRemoveFromDictionary); + var idRemoval = removeRedactionRepository.findByIdAndNotSoftDeleted(new AnnotationEntityId(annotationId, fileId)) + .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); + + idRemoval.setStatus(annotationStatus); + idRemoval.setRemoveFromDictionary(isAddOrRemoveFromDictionary); + idRemoval.setTypeIdsOfModifiedDictionaries(typeIdsOfModifiedDictionaries); + + removeRedactionRepository.saveAndFlush(idRemoval); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java index a14912740..29ee81d1a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java @@ -23,12 +23,6 @@ public interface RemoveRedactionRepository extends JpaRepository findByIdAndNotSoftDeleted(AnnotationEntityId annotationEntityId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java index 7cf59faf6..6e2d09f64 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java @@ -25,6 +25,9 @@ public interface EntryRepository extends EntryRepositoryCustom, JpaRepository findByTypeIdAndVersionGreaterThan(String typeId, long version); + List findByTypeIdContainsAndValue(String typeId, String value); + + @Modifying(flushAutomatically = true, clearAutomatically = true) @Transactional @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 56693619d..c3e7442bc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -137,3 +137,7 @@ databaseChangeLog: file: db/changelog/tenant/50-add-file-status-error-info.yaml - include: file: db/changelog/tenant/51-add-dossier-dictionary-only-flag.yaml + - include: + file: db/changelog/tenant/103-added-remove-from-all-dossier-to-manual-remove-redaction.yaml + - include: + file: db/changelog/tenant/104-added-add-to-all-dossier-to-manual-add-redaction.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/103-added-remove-from-all-dossier-to-manual-remove-redaction.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/103-added-remove-from-all-dossier-to-manual-remove-redaction.yaml new file mode 100644 index 000000000..45a45f2b7 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/103-added-remove-from-all-dossier-to-manual-remove-redaction.yaml @@ -0,0 +1,46 @@ +databaseChangeLog: + - changeSet: + id: added-remove-from-all-dossier-to-manual-remove-redaction + author: dom + changes: + - addColumn: + columns: + - column: + name: remove_from_all_dossiers + type: BOOLEAN + tableName: id_removal + - changeSet: + id: added-remove-from-all-dossier-to-manual-remove-redaction-2 + author: dom + changes: + - createTable: + columns: + - column: + constraints: + nullable: false + name: id_removal_entity_annotation_id + type: VARCHAR(255) + - column: + constraints: + nullable: false + name: id_removal_entity_file_id + type: VARCHAR(255) + - column: + name: type_ids_of_modified_dictionaries + type: VARCHAR(255) + tableName: id_removal_entity_type_ids_of_modified_dictionaries + - changeSet: + id: added-remove-from-all-dossier-to-manual-remove-redaction-3 + author: dom + changes: + - addForeignKeyConstraint: + baseColumnNames: id_removal_entity_annotation_id, id_removal_entity_file_id + baseTableName: id_removal_entity_type_ids_of_modified_dictionaries + constraintName: fk64e6qfq7tqad3lknqmqpzzz + deferrable: false + initiallyDeferred: false + onDelete: NO ACTION + onUpdate: NO ACTION + referencedColumnNames: annotation_id, file_id + referencedTableName: id_removal + validate: true \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-added-add-to-all-dossier-to-manual-add-redaction.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-added-add-to-all-dossier-to-manual-add-redaction.yaml new file mode 100644 index 000000000..7b1d1c1ac --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/104-added-add-to-all-dossier-to-manual-add-redaction.yaml @@ -0,0 +1,46 @@ +databaseChangeLog: + - changeSet: + id: added-add-to-all-dossier-to-manual-add-redaction + author: dom + changes: + - addColumn: + columns: + - column: + name: add_to_all_dossiers + type: BOOLEAN + tableName: manual_redaction + - changeSet: + id: added-add-to-all-dossier-to-manual-add-redaction-2 + author: dom + changes: + - createTable: + columns: + - column: + constraints: + nullable: false + name: manual_redaction_entry_entity_annotation_id + type: VARCHAR(255) + - column: + constraints: + nullable: false + name: manual_redaction_entry_entity_file_id + type: VARCHAR(255) + - column: + name: type_ids_of_modified_dictionaries + type: VARCHAR(255) + tableName: manual_redaction_entry_entity_type_ids_of_modified_dictionaries + - changeSet: + id: added-add-to-all-dossier-to-manual-add-redaction-3 + author: dom + changes: + - addForeignKeyConstraint: + baseColumnNames: manual_redaction_entry_entity_annotation_id, manual_redaction_entry_entity_file_id + baseTableName: manual_redaction_entry_entity_type_ids_of_modified_dictionaries + constraintName: fk64e6qfq7tqad3lknqmqpzaa + deferrable: false + initiallyDeferred: false + onDelete: NO ACTION + onUpdate: NO ACTION + referencedColumnNames: annotation_id, file_id + referencedTableName: manual_redaction + validate: true \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/client/InternalDictionaryClient.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/client/InternalDictionaryClient.java new file mode 100644 index 000000000..5f362f75d --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/client/InternalDictionaryClient.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.peristence.v1.server.integration.client; + +import org.springframework.cloud.openfeign.FeignClient; + +import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource; + +@FeignClient(name = "InternalDictionaryClient", url = "http://localhost:${server.port}") +public interface InternalDictionaryClient extends DictionaryResource { + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 658d471d6..a1622b13b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -4,14 +4,12 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.mock.web.MockMultipartFile; @@ -32,14 +30,12 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange; import com.iqser.red.service.persistence.service.v1.api.shared.model.ViewedPagesRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequest; @@ -344,7 +340,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { Set.of(AddRedactionRequest.builder() .addToDictionary(true) .type(type.getType()) - .addToDossierDictionary(false) + .addToAllDossiers(true) .reason("1") .value("test") .legalBasis("1") diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 13bb7c246..d27b655d4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; @@ -15,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; +import com.iqser.red.service.peristence.v1.server.integration.client.InternalDictionaryClient; import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; @@ -25,7 +27,9 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers import com.iqser.red.service.peristence.v1.server.integration.utils.MetricValidationUtils; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter; import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext; +import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; @@ -33,6 +37,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequest; @@ -83,6 +88,228 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @Autowired private UserProvider userProvider; + @Autowired + private InternalDictionaryClient internalDictionaryClient; + + + @Test + public void testAddToAllDossiersIfDeletedInOneDossier(){ + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate); + + //FIXME should be created on the fly. + CreateTypeValue dossierDictionaryType = MagicConverter.convert(type, CreateTypeValue.class); + dictionaryClient.addType(dossierDictionaryType, dossier.getDossierId()); + + dictionaryClient.deleteEntries(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), dossier.getDossierId(), DictionaryEntryType.ENTRY); + + var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId(), dossier.getDossierId()), null); + assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); + assertThat(dossierDictionary.getEntries().get(0).isDeleted()).isTrue(); + + manualRedactionClient.addRedactionBulk(dossier.getId(), + file.getId(), + Set.of(AddRedactionRequest.builder() + .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) + .section("section test") + .addToDictionary(true) + .addToAllDossiers(true) + .type(type.getType()) + .reason("1") + .value("Luke Skywalker") + .legalBasis("1") + .sourceId("SourceId") + .build())); + + dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId(), dossier.getDossierId()), null); + assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); + assertThat(dossierDictionary.getEntries().get(0).isDeleted()).isFalse(); + + var dossierTemplateDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId()), null); + assertThat(dossierTemplateDictionary.getEntries().size()).isEqualTo(1); + assertThat(dossierTemplateDictionary.getEntries().get(0).isDeleted()).isFalse(); + + + } + + @Test + public void testAddToDossierDictionary(){ + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate); + + //FIXME should be created on the fly. + CreateTypeValue dossierDictionaryType = MagicConverter.convert(type, CreateTypeValue.class); + dictionaryClient.addType(dossierDictionaryType, dossier.getDossierId()); + + manualRedactionClient.addRedactionBulk(dossier.getId(), + file.getId(), + Set.of(AddRedactionRequest.builder() + .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) + .section("section test") + .addToDictionary(true) + .addToAllDossiers(false) + .type(type.getType()) + .reason("1") + .value("Luke Skywalker") + .legalBasis("1") + .sourceId("SourceId") + .build())); + + var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId(), dossier.getDossierId()), null); + assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); + assertThat(dossierDictionary.getEntries().get(0).isDeleted()).isFalse(); + + var dossierTemplateDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId()), null); + assertThat(dossierTemplateDictionary.getEntries().isEmpty()).isTrue(); + } + + @Test + public void testManualRemoveFromAllDossiersAndUndo(){ + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate); + + var entries = new ArrayList(); + entries.add("Luke Skywalker"); + entries.add("Darth Vader"); + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY); + + //FIXME should be created on the fly. + CreateTypeValue dossierDictionaryType = MagicConverter.convert(type, CreateTypeValue.class); + dictionaryClient.addType(dossierDictionaryType, dossier.getDossierId()); + + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), false, dossier.getDossierId(), DictionaryEntryType.ENTRY); + + var dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker","Darth Vader"); + + var dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker"); + + var redactionLog = new RedactionLog(1, + 1, + List.of(RedactionLogEntry.builder().id("AnnotationId").type("test").value("Luke Skywalker").isDictionaryEntry(true).build()), + null, + 0, + 0, + 0, + 0); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog); + + when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog); + + manualRedactionClient.removeRedactionBulk(dossier.getId(), + file.getId(), + Set.of(RemoveRedactionRequest.builder().annotationId("AnnotationId").removeFromDictionary(true).removeFromAllDossiers(true).build())).get(0); + + dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Darth Vader"); + dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary.getEntries()).isEmpty(); + + manualRedactionClient.undo(dossier.getDossierId(), file.getFileId(), Set.of("AnnotationId")); + + dossierTemplateDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null); + assertThat(dossierTemplateDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker","Darth Vader"); + dossierDictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), dossier.getDossierId()); + assertThat(dossierDictionary.getEntries()).containsExactlyInAnyOrder("Luke Skywalker"); + } + + + @Test + public void testRemoveFromDossierIfOnlyDossierTemplateEntryExists(){ + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate); + + var entries = new ArrayList(); + entries.add("Luke Skywalker"); + entries.add("Darth Vader"); + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY); + + //FIXME should be created on the fly. + CreateTypeValue dossierDictionaryType = MagicConverter.convert(type, CreateTypeValue.class); + dictionaryClient.addType(dossierDictionaryType, dossier.getDossierId()); + + var redactionLog = new RedactionLog(1, + 1, + List.of(RedactionLogEntry.builder().id("AnnotationId").type("test").value("Luke Skywalker").isDictionaryEntry(true).build()), + null, + 0, + 0, + 0, + 0); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog); + + when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog); + + manualRedactionClient.removeRedactionBulk(dossier.getId(), + file.getId(), + Set.of(RemoveRedactionRequest.builder().annotationId("AnnotationId").removeFromDictionary(true).build())).get(0); + + var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId(), dossier.getDossierId()), null); + + + var dossierTemplateDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId()), null); + assertThat(dossierTemplateDictionary.getEntries().size()).isEqualTo(2); + assertThat(dossierTemplateDictionary.getEntries().stream().filter(e -> e.getValue().equals("Luke Skywalker")).findFirst().get().isDeleted()).isFalse(); + + assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); + assertThat(dossierDictionary.getEntries().get(0).isDeleted()).isTrue(); + } + + + @Test + public void testRemoveFromAllDossiersIfOnlyDossierEntryExists(){ + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate); + + //FIXME should be created on the fly. + CreateTypeValue dossierDictionaryType = MagicConverter.convert(type, CreateTypeValue.class); + dictionaryClient.addType(dossierDictionaryType, dossier.getDossierId()); + dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("Luke Skywalker"), false, dossier.getDossierId(), DictionaryEntryType.ENTRY); + + + var redactionLog = new RedactionLog(1, + 1, + List.of(RedactionLogEntry.builder().id("AnnotationId").type("test").value("Luke Skywalker").isDictionaryEntry(true).isDossierDictionaryEntry(true).build()), + null, + 0, + 0, + 0, + 0); + fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog); + + when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog); + + manualRedactionClient.removeRedactionBulk(dossier.getId(), + file.getId(), + Set.of(RemoveRedactionRequest.builder().annotationId("AnnotationId").removeFromDictionary(true).removeFromAllDossiers(true).build())).get(0); + + var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId(), dossier.getDossierId()), null); + assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); + assertThat(dossierDictionary.getEntries().get(0).isDeleted()).isTrue(); + + var dossierTemplateDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(dossierDictionaryType.getType(), dossierTemplate.getDossierTemplateId()), null); + assertThat(dossierTemplateDictionary.getEntries().size()).isEqualTo(0); + } + + + + @Test @SneakyThrows @@ -230,7 +457,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .positions(List.of(Rectangle.builder().page(100).topLeftY(1).topLeftX(1).height(1).width(1).build())) .section("section test") .addToDictionary(true) - .addToDossierDictionary(false) + .addToAllDossiers(true) .type(type.getType()) .reason("1") .value("test") @@ -248,7 +475,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) .section("section test") .addToDictionary(true) - .addToDossierDictionary(false) + .addToAllDossiers(true) .type(type.getType()) .reason("1") .value("test") @@ -289,7 +516,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { file.getId(), Set.of(AddRedactionRequest.builder() .addToDictionary(true) - .addToDossierDictionary(false) + .addToDossierDictionary(true) .type(type.getType()) .reason("1") .value("test") diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/AddRedactionRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/AddRedactionRequest.java index da90dd4e4..3d7f37771 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/AddRedactionRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/AddRedactionRequest.java @@ -17,17 +17,18 @@ import lombok.NoArgsConstructor; public class AddRedactionRequest { private String user; - private String typeId; + private String dossierTemplateTypeId; private String value; private String reason; private String legalBasis; private boolean addToDictionary; - private boolean addToDossierDictionary; + private boolean addToAllDossiers; private DictionaryEntryType dictionaryEntryType; private AnnotationStatus status; private String section; private boolean rectangle; private String sourceId; + private String dossierId; @Builder.Default private List positions = new ArrayList<>(); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RemoveRedactionRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RemoveRedactionRequest.java index 1319251df..eb94b5453 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RemoveRedactionRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RemoveRedactionRequest.java @@ -15,6 +15,7 @@ public class RemoveRedactionRequest { private String user; private AnnotationStatus status; private boolean removeFromDictionary; + private boolean removeFromAllDossiers; private String comment; private int page; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/IdRemoval.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/IdRemoval.java index 924af6ba6..fac014413 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/IdRemoval.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/IdRemoval.java @@ -15,6 +15,7 @@ import lombok.NoArgsConstructor; public class IdRemoval extends BaseAnnotation { private boolean removeFromDictionary; + private boolean removeFromAllDossiers; @Builder @@ -25,10 +26,12 @@ public class IdRemoval extends BaseAnnotation { OffsetDateTime requestDate, OffsetDateTime processedDate, OffsetDateTime softDeletedTime, - boolean removeFromDictionary) { + boolean removeFromDictionary, + boolean removeFromAllDossiers) { super(annotationId, fileId, user, status, requestDate, processedDate, softDeletedTime); this.removeFromDictionary = removeFromDictionary; + this.removeFromAllDossiers = removeFromAllDossiers; } } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionRequest.java index 6d538372d..9719340f7 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionRequest.java @@ -30,6 +30,7 @@ public class AddRedactionRequest { private String legalBasis; private boolean addToDictionary; + private boolean addToAllDossiers; @NonNull @Builder.Default @@ -37,6 +38,7 @@ public class AddRedactionRequest { private AddCommentRequest comment; + @Deprecated // Do not use this anymore. will be removed. private boolean addToDossierDictionary; private boolean forceAddToDictionary; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionRequest.java index 8075d2654..668c2cd94 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionRequest.java @@ -16,6 +16,7 @@ public class RemoveRedactionRequest { private String annotationId; private boolean removeFromDictionary; + private boolean removeFromAllDossiers; private String comment; -- 2.47.2