diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java index caa05809c..77c42054a 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java @@ -19,15 +19,13 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService; import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; -import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; -import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionUndoService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.PendingEntryFactory; @@ -40,11 +38,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.CommentResp import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntryResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationComments; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.CommentRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; @@ -190,7 +189,7 @@ public class ManualRedactionController implements ManualRedactionResource { addRedactionRequests.stream() .anyMatch(AddRedactionRequestModel::isAddToAllDossiers)); - List responseList = manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests, dossier); + List responseList = manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequests, dossier); responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) .objectId(fileId) @@ -199,7 +198,7 @@ public class ManualRedactionController implements ManualRedactionResource { .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId())) .build())); - return ManualRedactionResponse.builder().manualAddResponses(responseList).build(); + return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build(); } @@ -212,15 +211,15 @@ public class ManualRedactionController implements ManualRedactionResource { var dossier = dossierManagementService.getDossierById(dossierId, false, false); dictionaryPersistenceService.getType(TypeIdUtils.toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId())); - fileStatusService.setStatusBulkLocalRedactionsProcessing(dossierId, fileId, addRedactionRequest); + fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest); EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest); return ManualRedactionResponse.builder() - .manualAddResponses(List.of(ManualAddResponse.builder() - .annotationId(manualRedactionService.getPendingBulkLocalAnnotationId(fileId, addRedactionRequest)) - .entityLogEntry(entityLogEntry) - .build())) + .manualAnnotationResponses(List.of(ManualAnnotationResponse.builder() + .annotationId(manualRedactionService.getNewAnnotationId()) + .entityLogEntry(entityLogEntry) + .build())) .build(); } @@ -238,11 +237,11 @@ public class ManualRedactionController implements ManualRedactionResource { removeRedactionRequests.stream() .anyMatch(RemoveRedactionRequestModel::isRemoveFromAllDossiers)); - List responseList = manualRedactionService.addRemoveRedaction(dossierId, - fileId, - removeRedactionRequests, - dossier.getDossierTemplateId(), - includeUnprocessed); + List responseList = manualRedactionService.addRemoveRedaction(dossierId, + fileId, + removeRedactionRequests, + dossier.getDossierTemplateId(), + includeUnprocessed); responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) @@ -252,7 +251,7 @@ public class ManualRedactionController implements ManualRedactionResource { .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId())) .build())); - return ManualRedactionResponse.builder().manualAddResponses(responseList).build(); + return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build(); } @@ -263,20 +262,23 @@ public class ManualRedactionController implements ManualRedactionResource { boolean includeUnprocessed) { verifyAccess(dossierId, fileId); + verifyRequest(removeRedactionRequest.isRectangle(), removeRedactionRequest.getPosition(), removeRedactionRequest.getValue()); Set removeRedactionRequestModels; FileModel status = fileStatusService.getStatus(fileId); if (!status.isExcludedFromAutomaticAnalysis()) { - Set entryIds = getFilteredEntityLogEntryIds(removeRedactionRequest.isRectangle(), - removeRedactionRequest.getValue(), - removeRedactionRequest.isCaseSensitive(), - removeRedactionRequest.getOriginTypes(), - removeRedactionRequest.getOriginLegalBases(), - removeRedactionRequest.getPageNumbers(), - removeRedactionRequest.getPosition()); - removeRedactionRequestModels = entryIds.stream() - .map(entryId -> RemoveRedactionRequestModel.builder().annotationId(entryId).build()) + Set entries = getFilteredEntityLogEntries(dossierId, + fileId, + removeRedactionRequest.isRectangle(), + removeRedactionRequest.getValue(), + removeRedactionRequest.isCaseSensitive(), + removeRedactionRequest.getOriginTypes(), + removeRedactionRequest.getOriginLegalBases(), + removeRedactionRequest.getPageNumbers(), + removeRedactionRequest.getPosition()); + removeRedactionRequestModels = entries.stream() + .map(entry -> RemoveRedactionRequestModel.builder().annotationId(entry.getId()).build()) .collect(Collectors.toSet()); } else { @@ -293,7 +295,7 @@ public class ManualRedactionController implements ManualRedactionResource { removeRedactionRequest.getPosition()); removeRedactionRequestModels = filteredEntityLogResponses.stream() - .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).build()) + .map(entityLogEntry -> RemoveRedactionRequestModel.builder().annotationId(entityLogEntry.getId()).comment(removeRedactionRequest.getComment()).build()) .collect(Collectors.toSet()); } return removeRedactionBulk(dossierId, fileId, removeRedactionRequestModels, includeUnprocessed); @@ -307,7 +309,7 @@ public class ManualRedactionController implements ManualRedactionResource { verifyAccessAndDossierExistence(dossierId, fileId); - List responseList = manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequests); + List responseList = manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequests); responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) @@ -317,7 +319,7 @@ public class ManualRedactionController implements ManualRedactionResource { .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId())) .build())); - return ManualRedactionResponse.builder().manualAddResponses(responseList).build(); + return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build(); } @@ -328,7 +330,7 @@ public class ManualRedactionController implements ManualRedactionResource { verifyAccessAndDossierExistence(dossierId, fileId); - List responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequests); + List responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequests); responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) @@ -338,7 +340,7 @@ public class ManualRedactionController implements ManualRedactionResource { .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId())) .build())); - return ManualRedactionResponse.builder().manualAddResponses(responseList).build(); + return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build(); } @@ -351,7 +353,7 @@ public class ManualRedactionController implements ManualRedactionResource { var dossier = dossierManagementService.getDossierById(dossierId, false, false); verifyAccess(dossierId, fileId); - List responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, includeUnprocessed); + List responseList = manualRedactionService.addRecategorization(dossierId, fileId, dossier, recategorizationRequests, includeUnprocessed); responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) @@ -361,7 +363,7 @@ public class ManualRedactionController implements ManualRedactionResource { .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId())) .build())); - return ManualRedactionResponse.builder().manualAddResponses(responseList).build(); + return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build(); } @@ -372,25 +374,28 @@ public class ManualRedactionController implements ManualRedactionResource { boolean includeUnprocessed) { verifyAccess(dossierId, fileId); + verifyRequest(recategorizationRequest.isRectangle(), recategorizationRequest.getPosition(), recategorizationRequest.getValue()); Set recategorizationRequestModels; FileModel status = fileStatusService.getStatus(fileId); if (!status.isExcludedFromAutomaticAnalysis()) { - Set entryIds = getFilteredEntityLogEntryIds(recategorizationRequest.isRectangle(), - recategorizationRequest.getValue(), - recategorizationRequest.isCaseSensitive(), - recategorizationRequest.getOriginTypes(), - recategorizationRequest.getOriginLegalBases(), - recategorizationRequest.getPageNumbers(), - recategorizationRequest.getPosition()); + Set entries = getFilteredEntityLogEntries(dossierId, + fileId, + recategorizationRequest.isRectangle(), + recategorizationRequest.getValue(), + recategorizationRequest.isCaseSensitive(), + recategorizationRequest.getOriginTypes(), + recategorizationRequest.getOriginLegalBases(), + recategorizationRequest.getPageNumbers(), + recategorizationRequest.getPosition()); - recategorizationRequestModels = entryIds.stream() - .map(entryId -> RecategorizationRequestModel.builder() - .annotationId(entryId) - .type(recategorizationRequest.getType()) + recategorizationRequestModels = entries.stream() + .map(entry -> RecategorizationRequestModel.builder() + .annotationId(entry.getId()) + .type(recategorizationRequest.isRectangle() ? entry.getType() : recategorizationRequest.getType()) .legalBasis(recategorizationRequest.getLegalBasis()) .section(recategorizationRequest.getSection()) - .value(recategorizationRequest.getValue()) + .value(entry.getValue()) .build()) .collect(Collectors.toSet()); @@ -410,10 +415,11 @@ public class ManualRedactionController implements ManualRedactionResource { recategorizationRequestModels = filteredEntityLogResponses.stream() .map(entityLogEntry -> RecategorizationRequestModel.builder() .annotationId(entityLogEntry.getId()) - .type(recategorizationRequest.getType()) + .type(recategorizationRequest.isRectangle() ? entityLogEntry.getType() : recategorizationRequest.getType()) .legalBasis(recategorizationRequest.getLegalBasis()) .section(recategorizationRequest.getSection()) - .value(recategorizationRequest.getValue()) + .value(entityLogEntry.getValue()) + .comment(recategorizationRequest.getComment()) .build()) .collect(Collectors.toSet()); } @@ -430,7 +436,7 @@ public class ManualRedactionController implements ManualRedactionResource { verifyAccessAndDossierExistence(dossierId, fileId); - List responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, includeUnprocessed); + List responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, resizeRedactionRequests, includeUnprocessed); responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) @@ -440,25 +446,27 @@ public class ManualRedactionController implements ManualRedactionResource { .details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId())) .build())); - return ManualRedactionResponse.builder().manualAddResponses(responseList).build(); + return ManualRedactionResponse.builder().manualAnnotationResponses(responseList).build(); } - private Set getFilteredEntityLogEntryIds(boolean rectangle, - String value, - boolean caseSensitive, - Set originTypes, - Set originLegalBases, - Set pageNumbers, - Position position) { + private Set getFilteredEntityLogEntries(String dossierId, + String fileId, + boolean rectangle, + String value, + boolean caseSensitive, + Set originTypes, + Set originLegalBases, + Set pageNumbers, + Position position) { - Set entryIds; + Set entries; if (!rectangle) { - entryIds = entityLogMongoService.findEntryIdsByValueWithFilters(value, caseSensitive, originTypes, originLegalBases, pageNumbers); + entries = entityLogMongoService.findEntriesByValueWithFilters(dossierId, fileId, value, caseSensitive, originTypes, originLegalBases, pageNumbers); } else { - entryIds = entityLogMongoService.findEntryIdsByMatchingFullPositionWithFilters(position.getRectangle(), originTypes, originLegalBases, pageNumbers); + entries = entityLogMongoService.findEntriesByMatchingFullPositionWithFilters(dossierId, fileId, position.getRectangle(), originTypes, originLegalBases, pageNumbers); } - return entryIds; + return entries; } @@ -496,6 +504,7 @@ public class ManualRedactionController implements ManualRedactionResource { Set pageNumbers) { return entries.stream() + .filter(entry -> !entry.getEntryType().equals(EntryType.AREA)) .filter(entry -> filterByValue(entry, value, caseSensitive)) .filter(entry -> filterByOriginType(entry, originTypes)) .filter(entry -> filterByLegalBases(entry, originLegalBases)) @@ -511,6 +520,7 @@ public class ManualRedactionController implements ManualRedactionResource { Set pageNumbers) { return entries.stream() + .filter(entry -> entry.getEntryType().equals(EntryType.AREA)) .filter(entry -> filterByRectangle(entry, rectangle)) .filter(entry -> filterByOriginType(entry, originTypes)) .filter(entry -> filterByLegalBases(entry, originLegalBases)) @@ -563,6 +573,19 @@ public class ManualRedactionController implements ManualRedactionResource { } + private void verifyRequest(boolean isRectangle, Position position, String value) { + + if (isRectangle && position == null) { + throw new BadRequestException("Position must be set for rectangle annotations."); + } + + if (!isRectangle && value == null) { + throw new BadRequestException("Value must be set."); + } + + } + + private void verifyAccessForDossier(String dossierId, String fileId, boolean allDossiersAffected) { accessControlService.checkAccessPermissionsToDossier(dossierId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AccessControlService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AccessControlService.java index 19dcfa13b..36def5563 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AccessControlService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AccessControlService.java @@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Dossi import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; @@ -142,8 +143,13 @@ public class AccessControlService { public void verifyFileIsNotApproved(String dossierId, String fileId) { try { - var status = fileStatusManagementService.getFileStatus(fileId, false).getWorkflowStatus(); + FileModel fileStatus = fileStatusManagementService.getFileStatus(fileId, false); + if(!fileStatus.getDossierId().equals(dossierId)) { + throw new NotFoundException("File " + fileId + " dossier " + dossierId + " not found."); + } + + var status = fileStatus.getWorkflowStatus(); if (WorkflowStatus.APPROVED.equals(status)) { throw new NotAllowedException("File is already in status APPROVED."); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 5bbefbfb1..233f49e56 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -1020,7 +1020,7 @@ public class FileStatusService { @Transactional - public void setStatusBulkLocalRedactionsProcessing(String dossierId, String fileId, AddRedactionBulkLocalRequestModel addRedactionBulkLocalRequestModel) { + public void setStatusBulkLocalRedactionsProcessing(String fileId, AddRedactionBulkLocalRequestModel addRedactionBulkLocalRequestModel) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -1038,6 +1038,7 @@ public class FileStatusService { .legalBasis(addRedactionBulkLocalRequestModel.getLegalBasis()) .section(addRedactionBulkLocalRequestModel.getSection()) .pageNumbers(addRedactionBulkLocalRequestModel.getPageNumbers()) + .comment(addRedactionBulkLocalRequestModel.getComment()) .build(); addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java index c9cf60349..5ce183cbc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java @@ -3,13 +3,13 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual import static com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionMapper.getDictionaryEntryType; import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert; -import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -57,7 +57,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; @@ -71,7 +71,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel; @@ -117,17 +116,18 @@ public class ManualRedactionService { @Transactional @Observed(name = "ManualRedactionService", contextualName = "add-manual-redaction") - public List addAddRedaction(String dossierId, String fileId, Set addRedactionRequests, Dossier dossier) { + public List addAddRedaction(String dossierId, String fileId, Set addRedactionRequests, Dossier dossier) { if (addRedactionRequests.isEmpty()) { throw new BadRequestException("add redaction requests is empty for request: dossierId:" + dossierId + " fileId:" + fileId); } - var response = new ArrayList(); + var response = new ArrayList(); List manualRedactionEntryEntities = new ArrayList<>(); List requests = manualRedactionMapper.toAddRedactionRequestList(dossierId, addRedactionRequests, dossier); var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId); + FileModel fileStatus = fileStatusService.getStatus(fileId); for (AddRedactionRequest addRedactionRequest : requests) { @@ -136,7 +136,7 @@ public class ManualRedactionService { manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue()); validatePositions(fileId, addRedactionRequest); - String annotationId = hashString(fileId + addRedactionRequest); + String annotationId = getNewAnnotationId(); ManualRedactionEntryEntity manualRedactionEntryEntity = addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); manualRedactionEntryEntities.add(manualRedactionEntryEntity); @@ -160,14 +160,13 @@ public class ManualRedactionService { new ManualRedactionEntryMapper())); } Long commentId = commentService.addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()); - response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).entityLogEntry(entityLogEntry).build()); + response.add(ManualAnnotationResponse.builder().annotationId(annotationId).commentId(commentId).entityLogEntry(entityLogEntry).build()); } var localManualRedactionEntryEntities = manualRedactionEntryEntities.stream() .filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToAllDossiers()) .collect(Collectors.toList()); - FileModel fileStatus = fileStatusService.getStatus(fileId); if (!localManualRedactionEntryEntities.isEmpty() && fileStatus.isExcludedFromAutomaticAnalysis()) { ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(convertEntriesToAdd(localManualRedactionEntryEntities)).build(); sendToSurroundingTextAnalysisQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions); @@ -197,11 +196,11 @@ public class ManualRedactionService { @Transactional - public List addRemoveRedaction(String dossierId, - String fileId, - Set removeRedactionRequests, - String dossierTemplateId, - boolean includeUnprocessed) { + public List addRemoveRedaction(String dossierId, + String fileId, + Set removeRedactionRequests, + String dossierTemplateId, + boolean includeUnprocessed) { if (removeRedactionRequests.isEmpty()) { throw new BadRequestException("remove redaction requests is empty for request: dossierId:" + dossierId + " fileId:" + fileId); @@ -212,7 +211,7 @@ public class ManualRedactionService { if (numberOfDictRemoves > 100) { throw new BadRequestException("Maximum number of remove from dictionary requests is 100."); } - var response = new ArrayList(); + var response = new ArrayList(); List> requests = manualRedactionMapper.toRemoveRedactionRequestList(dossierId, fileId, dossierTemplateId, @@ -257,7 +256,7 @@ public class ManualRedactionService { } else { entityLogEntry = pendingEntryFactory.buildRemoveFromDictionary(MagicConverter.convert(idRemoval, IdRemoval.class), entityLogEntry); } - response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); + response.add(ManualAnnotationResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); } reprocess(dossierId, fileId); @@ -287,12 +286,12 @@ public class ManualRedactionService { @Transactional - public List addForceRedaction(String dossierId, String fileId, Set forceRedactionRequests) { + public List addForceRedaction(String dossierId, String fileId, Set forceRedactionRequests) { if (forceRedactionRequests.isEmpty()) { throw new BadRequestException("force redaction requests is empty for request: dossierId:" + dossierId + " fileId:" + fileId); } - var response = new ArrayList(); + var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); List> requests = manualRedactionMapper.toForceRedactionRequestList(dossierId, fileId, @@ -312,7 +311,7 @@ public class ManualRedactionService { entityLogMergeService.mergeForceRedaction(MagicConverter.convert(manualForceRedaction, ManualForceRedaction.class), entityLogEntry, getAnalysisNumber(dossierId, fileId)); - response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); + response.add(ManualAnnotationResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); } reprocess(dossierId, fileId); @@ -324,13 +323,13 @@ public class ManualRedactionService { @Transactional - public List addLegalBasisChange(String dossierId, String fileId, Set legalBasisChangeRequests) { + public List addLegalBasisChange(String dossierId, String fileId, Set legalBasisChangeRequests) { if (legalBasisChangeRequests.isEmpty()) { throw new BadRequestException("legal absis change requests is empty for request: dossierId:" + dossierId + " fileId:" + fileId); } - var response = new ArrayList(); + var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); List> requests = manualRedactionMapper.toLegalBasisChangeRequestList(dossierId, fileId, @@ -350,7 +349,7 @@ public class ManualRedactionService { entityLogMergeService.mergeLegalBasisChange(MagicConverter.convert(manualLegalBasisChange, ManualLegalBasisChange.class), entityLogEntry, getAnalysisNumber(dossierId, fileId)); - response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); + response.add(ManualAnnotationResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); } reprocess(dossierId, fileId); fileStatusPersistenceService.setLastManualChangeDate(fileId, OffsetDateTime.now()); @@ -360,17 +359,17 @@ public class ManualRedactionService { @Transactional - public List addRecategorization(String dossierId, - String fileId, - Dossier dossier, - Set recategorizationRequests, - boolean includeUnprocessed) { + public List addRecategorization(String dossierId, + String fileId, + Dossier dossier, + Set recategorizationRequests, + boolean includeUnprocessed) { if (recategorizationRequests.isEmpty()) { throw new BadRequestException("recat redaction requests is empty for request: dossierId:" + dossierId + " fileId:" + fileId); } - var response = new ArrayList(); + var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId); List> requests = manualRedactionMapper.toRecategorizationRequestList(dossierId, @@ -418,7 +417,7 @@ public class ManualRedactionService { ManualRecategorization.class, new ManualRecategorizationMapper()), entityLogEntry); } - response.add(ManualAddResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); + response.add(ManualAnnotationResponse.builder().annotationId(recategorizationRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); recategorizationPersistenceService.saveAndFlush(recategorizationEntity); } @@ -433,13 +432,16 @@ public class ManualRedactionService { @Transactional @Observed(name = "ManualRedactionService", contextualName = "add-manual-redaction") - public List addResizeRedaction(String dossierId, String fileId, Set resizeRedactionRequests, boolean includeUnprocessed) { + public List addResizeRedaction(String dossierId, + String fileId, + Set resizeRedactionRequests, + boolean includeUnprocessed) { if (resizeRedactionRequests.isEmpty()) { throw new BadRequestException("resize redaction requests is empty for request: dossierId:" + dossierId + " fileId:" + fileId); } - List response = new ArrayList<>(); + List response = new ArrayList<>(); List manualResizeRedactionEntities = new ArrayList<>(); List> requests = manualRedactionMapper.toResizeRedactionRequestList(dossierId, fileId, @@ -479,7 +481,7 @@ public class ManualRedactionService { ManualResizeRedaction.class, new ManualResizeRedactionMapper()), entityLogEntry); } - response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); + response.add(ManualAnnotationResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).entityLogEntry(entityLogEntry).build()); } manualResizeRedactionEntities = manualResizeRedactionEntities.stream() @@ -701,15 +703,9 @@ public class ManualRedactionService { } - public String getPendingBulkLocalAnnotationId(String fileId, AddRedactionBulkLocalRequestModel addRedactionBulkLocalRequestModel) { + public String getNewAnnotationId() { - return hashString(fileId + addRedactionBulkLocalRequestModel); - } - - - private String hashString(String input) { - - return hashFunction.hashString(input, StandardCharsets.UTF_8).toString(); + return UUID.randomUUID().toString(); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java index 1fc3e29f1..f501d7591 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java @@ -4,6 +4,7 @@ import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import org.springframework.beans.BeanUtils; @@ -12,6 +13,7 @@ import org.springframework.transaction.annotation.Transactional; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository; @@ -47,29 +49,38 @@ public class RemoveRedactionPersistenceService { .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); } + public List findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) { if (options.getAnnotationIds().isEmpty()) { return removeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges()); } - return removeRedactionRepository.findByFileIdAndOptionsAndAnnotationIds(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges(), options.getAnnotationIds()); + return removeRedactionRepository.findByFileIdAndOptionsAndAnnotationIds(fileId, + options.isIncludeDeletions(), + options.isIncludeOnlyUnprocessed(), + options.isIncludeDictChanges(), + options.getAnnotationIds()); } + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { return removeRedactionRepository.softDeleteByFileIds(fileIds, softDeletionTime); } + public int deleteByFileIds(List fileIds) { return removeRedactionRepository.deleteByFileIds(fileIds); } + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { removeRedactionRepository.undeleteByFileId(fileId, deletionTime); } + @Transactional public void hardDelete(String fileId, String annotationId) { @@ -110,4 +121,10 @@ public class RemoveRedactionPersistenceService { removeRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } + + public Optional findById(String annotationId, String fileId) { + + return removeRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId)); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java index c9a3c129c..0ba4d4c5b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java @@ -14,7 +14,6 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; @@ -24,10 +23,11 @@ import com.iqser.red.service.persistence.management.v1.processor.service.websock import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory; import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; @@ -75,39 +75,41 @@ public class SearchTermOccurrencesResponseReceiver { var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false); - Set addRedactionRequests = response.getEntityPositions() + Set addRedactionRequests = response.getFoundTerms() .stream() - .map(entityPosition -> AddRedactionRequestModel.builder() + .map(term -> AddRedactionRequestModel.builder() .type(response.getType()) - .value(response.getSearchTerm()) + .value(term.value()) .reason(response.getReason()) .legalBasis(response.getLegalBasis()) - .positions(convertPositions(entityPosition.positions())) + .positions(convertPositions(term.positions())) .section(response.getSection()) + .comment(response.getComment() == null ? null : new AddCommentRequestModel(response.getComment())) .build()) .collect(Collectors.toSet()); log.info("Received manual redaction requests for file {} in dossier {} after term search", response.getFileId(), dossier.getId()); - List manualAddResponses = manualRedactionService.addAddRedaction(response.getDossierId(), response.getFileId(), addRedactionRequests, dossier); - manualAddResponses.forEach(manualAddResponse -> auditPersistenceService.audit(AuditRequest.builder() - .userId(KeycloakSecurity.getUserId()) - .objectId(response.getFileId()) - .category(AuditCategory.DOCUMENT.name()) - .message("Manual annotation was added.") - .details(Map.of("dossierId", - response.getDossierId(), - "fileId", - response.getFileId(), - "annotationId", - manualAddResponse.getAnnotationId())) - .build())); + List manualAnnotationResponses = manualRedactionService.addAddRedaction(response.getDossierId(), + response.getFileId(), + addRedactionRequests, + dossier); + manualAnnotationResponses.forEach(manualAddResponse -> auditPersistenceService.audit(AuditRequest.builder() + .userId(KeycloakSecurity.getUserId()) + .objectId(response.getFileId()) + .category(AuditCategory.DOCUMENT.name()) + .message("Manual annotation was added.") + .details(Map.of("dossierId", + response.getDossierId(), + "fileId", + response.getFileId(), + "annotationId", + manualAddResponse.getAnnotationId())) + .build())); fileStatusPersistenceService.updateProcessingStatus(response.getFileId(), ProcessingStatus.PROCESSED); int analysisVersion = fileStatusPersistenceService.getStatus(response.getFileId()).getAnalysisVersion(); - webSocketService.sendAnalysisEvent(response.getDossierId(), - response.getFileId(), - AnalyseStatus.BULK_LOCAL_REDACTIONS_PROCESSED, analysisVersion); + webSocketService.sendAnalysisEvent(response.getDossierId(), response.getFileId(), AnalyseStatus.BULK_LOCAL_REDACTIONS_PROCESSED, analysisVersion); log.info("Finished adding all manual redactions, the file is processed again"); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMongoServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMongoServiceTest.java index 85e598b3f..e004425ab 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMongoServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMongoServiceTest.java @@ -343,33 +343,36 @@ public class EntityLogMongoServiceTest extends AbstractPersistenceServerServiceT entityLogMongoService.saveEntityLog(TEST_DOSSIER_ID, TEST_FILE1_ID, entityLog); String akessonValue = "Akesson"; - Set akessonSet = entityLogMongoService.findEntryIdsOfManualsWithValue(akessonValue); - assertEquals(akessonSet.size(), 3); - float[] akessonRectangle = {322.63608f, 308.1f, 41.375977f, 12.642f}; - Set akessonSetByPositions = entityLogMongoService.findEntryIdsOfManualsWithPositionRectangle(akessonRectangle); - assertEquals(akessonSetByPositions.size(), 3); - Set akessonSetFilterQuery = entityLogMongoService.findEntryIdsByMatchingFullPositionWithFilters(akessonRectangle, + Set akessonSetFilterQuery = entityLogMongoService.findEntryIdsByMatchingFullPositionWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + akessonRectangle, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); assertEquals(akessonSetFilterQuery.size(), 3); - Set akessonSetFilterQueryByPositions = entityLogMongoService.findEntryIdsByValueWithFilters(akessonValue, + Set akessonSetFilterQueryByPositions = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + akessonValue, false, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); assertEquals(akessonSetFilterQueryByPositions.size(), 3); - akessonSetFilterQuery = entityLogMongoService.findEntryIdsByMatchingFullPositionWithFilters(akessonRectangle, + akessonSetFilterQuery = entityLogMongoService.findEntryIdsByMatchingFullPositionWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + akessonRectangle, Collections.emptySet(), Collections.emptySet(), Set.of(1, 3)); assertEquals(akessonSetFilterQuery.size(), 2); - akessonSetFilterQueryByPositions = entityLogMongoService.findEntryIdsByValueWithFilters(akessonValue, + akessonSetFilterQueryByPositions = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + akessonValue, false, Collections.emptySet(), Collections.emptySet(), @@ -377,52 +380,65 @@ public class EntityLogMongoServiceTest extends AbstractPersistenceServerServiceT assertEquals(akessonSetFilterQueryByPositions.size(), 2); String baldridgeValue = "Baldridge"; - Set baldridgeSet = entityLogMongoService.findEntryIdsByValueWithFilters(baldridgeValue, + Set baldridgeSet = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + baldridgeValue, false, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); assertEquals(baldridgeSet.size(), 4); - Set baldridgeSetWithTypeFilter = entityLogMongoService.findEntryIdsByValueWithFilters(baldridgeValue, + Set baldridgeSetWithTypeFilter = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + baldridgeValue, false, Set.of("CBI_author", "Test"), Collections.emptySet(), Collections.emptySet()); assertEquals(baldridgeSetWithTypeFilter.size(), 2); - Set baldridgeSetWithLegalBasesFilter = entityLogMongoService.findEntryIdsByValueWithFilters(baldridgeValue, + Set baldridgeSetWithLegalBasesFilter = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + baldridgeValue, false, Set.of("CBI_author", "Test"), Set.of("Test"), Collections.emptySet()); assertEquals(baldridgeSetWithLegalBasesFilter.size(), 1); - String albaValueCapitalA = "Alba"; - Set albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(albaValueCapitalA, + Set albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + albaValueCapitalA, true, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); assertEquals(albaSet.size(), 4); - albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(albaValueCapitalA, - false, - Collections.emptySet(), - Collections.emptySet(), - Collections.emptySet()); - assertEquals(albaSet.size(), 4); - - String albaValue = "alba"; - albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(albaValue, + albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + albaValueCapitalA, false, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); assertEquals(albaSet.size(), 4); - albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(albaValue, + String albaValue = "alba"; + albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + albaValue, + false, + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet()); + assertEquals(albaSet.size(), 4); + + albaSet = entityLogMongoService.findEntryIdsByValueWithFilters(TEST_DOSSIER_ID, + TEST_FILE1_ID, + albaValue, true, Collections.emptySet(), Collections.emptySet(), diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 05a93a56c..27cb1fc14 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -811,7 +811,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0); AnnotationComments annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, fileId, - recatResponse.getManualAddResponses() + recatResponse.getManualAnnotationResponses() .get(0).getAnnotationId()); assertThat(annotationCommentsForManualRecat.getComments()).hasSize(1); @@ -830,7 +830,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0); annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, fileId, - recatResponse.getManualAddResponses() + recatResponse.getManualAnnotationResponses() .get(0).getAnnotationId()); assertThat(annotationCommentsForManualRecat.getComments()).hasSize(0); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 0208bbee2..bff43f39a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -6,11 +6,13 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -19,11 +21,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; +import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; import com.fasterxml.jackson.databind.ObjectMapper; @@ -38,14 +43,25 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions; import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; +import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.queue.SearchTermOccurrencesResponseReceiver; import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FoundTerm; import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; @@ -57,10 +73,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.Dictionary; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; @@ -128,6 +146,18 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @Autowired private ManualRedactionService manualRedactionService; + @Autowired + private ManualRedactionProviderService manualRedactionProviderService; + + @Autowired + private EntityLogMergeService entityLogMergeService; + + @Autowired + private DossierService dossierService; + + @Autowired + private SearchTermOccurrencesResponseReceiver searchTermOccurrencesResponseReceiver; + @Test public void testRemoveToDossierTemplateWithDossierDictionaryOnlyTrue() { @@ -417,7 +447,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()), - false).getManualAddResponses() + false).getManualAnnotationResponses() .get(0); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); @@ -476,7 +506,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .removeFromDictionary(true) .removeFromAllDossiers(true) .build()), - false).getManualAddResponses() + false).getManualAnnotationResponses() .get(0); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); @@ -540,7 +570,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier") @@ -549,7 +579,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") @@ -569,7 +599,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") @@ -588,7 +618,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier 1 var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses() + .annotationId(addRedactions.getManualAnnotationResponses() .get(0).getAnnotationId()) .comment("resized dossier redaction") .value("test redaction in dossier dictionary") @@ -709,7 +739,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier yayy") @@ -718,7 +748,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") @@ -738,7 +768,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") @@ -757,7 +787,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier 1 var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses() + .annotationId(addRedactions.getManualAnnotationResponses() .get(0).getAnnotationId()) .comment("resized dossier redaction") .value("test redaction in dossier") @@ -882,7 +912,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier") @@ -891,7 +921,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") @@ -911,7 +941,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") @@ -930,7 +960,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier dict var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses() + .annotationId(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .comment("resized dossier template redaction") .value("test redaction in dossier template dictionary") @@ -1052,7 +1082,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier yayy") @@ -1061,7 +1091,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") @@ -1081,7 +1111,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses() + .id(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") @@ -1100,7 +1130,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier dict var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses() + .annotationId(addRedactions.getManualAnnotationResponses() .get(1).getAnnotationId()) .comment("resized dossier template redaction") .value("test redaction in dossier template") @@ -1753,7 +1783,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse.getManualAddResponses() + .equals(forceResponse.getManualAnnotationResponses() .get(0).getAnnotationId()))); var unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); @@ -1764,7 +1794,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse.getManualAddResponses() + .equals(forceResponse.getManualAnnotationResponses() .get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), @@ -1793,7 +1823,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse.getManualAddResponses() + .equals(forceResponse.getManualAnnotationResponses() .get(0).getAnnotationId()))); assertFalse(allManualRedactions.getForceRedactions() .stream() @@ -1801,7 +1831,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse2.getManualAddResponses() + .equals(forceResponse2.getManualAnnotationResponses() .get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); @@ -1812,7 +1842,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(unprocessedManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse2.getManualAddResponses() + .equals(forceResponse2.getManualAnnotationResponses() .get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), @@ -1833,7 +1863,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse.getManualAddResponses() + .equals(forceResponse.getManualAnnotationResponses() .get(0).getAnnotationId()))); assertFalse(allManualRedactions.getForceRedactions() .stream() @@ -1841,7 +1871,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getForceRedactions() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(forceResponse2.getManualAddResponses() + .equals(forceResponse2.getManualAnnotationResponses() .get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); @@ -1902,7 +1932,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response.getManualAddResponses() + .equals(response.getManualAnnotationResponses() .get(0).getAnnotationId()))); var unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); @@ -1910,7 +1940,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(unprocessedManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response.getManualAddResponses() + .equals(response.getManualAnnotationResponses() .get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), @@ -1938,12 +1968,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response.getManualAddResponses() + .equals(response.getManualAnnotationResponses() .get(0).getAnnotationId()))); assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response2.getManualAddResponses() + .equals(response2.getManualAnnotationResponses() .get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); @@ -1951,12 +1981,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(unprocessedManualRedactions.getRecategorizations() .stream() .noneMatch(entry -> entry.getAnnotationId() - .equals(response.getManualAddResponses() + .equals(response.getManualAnnotationResponses() .get(0).getAnnotationId()))); assertTrue(unprocessedManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response2.getManualAddResponses() + .equals(response2.getManualAnnotationResponses() .get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), @@ -1974,12 +2004,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response.getManualAddResponses() + .equals(response.getManualAnnotationResponses() .get(0).getAnnotationId()))); assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response2.getManualAddResponses() + .equals(response2.getManualAnnotationResponses() .get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); @@ -2251,7 +2281,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .stream() .anyMatch(entry -> entry.getAnnotationId().equals("annotationId"))); - assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses() + assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAnnotationResponses() .get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long"); assertNull(allManualRedactions.getRecategorizations() @@ -2324,7 +2354,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getAnnotationId() - .equals(response.getManualAddResponses() + .equals(response.getManualAnnotationResponses() .get(0).getAnnotationId()))); assertTrue(allManualRedactions.getRecategorizations() .stream() @@ -2732,20 +2762,20 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - ManualAddResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(), - file.getId(), - Set.of(AddRedactionRequestModel.builder() - .sourceId("") - .reason("Reason") - .type(type.getType()) - .addToAllDossiers(false) - .addToDictionary(false) - .dictionaryEntryType(DictionaryEntryType.ENTRY) - .forceAddToDictionary(false) - .legalBasis("legalBasis") - .positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1))) - .value("MyValue") - .build())).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(), + file.getId(), + Set.of(AddRedactionRequestModel.builder() + .sourceId("") + .reason("Reason") + .type(type.getType()) + .addToAllDossiers(false) + .addToDictionary(false) + .dictionaryEntryType(DictionaryEntryType.ENTRY) + .forceAddToDictionary(false) + .legalBasis("legalBasis") + .positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1))) + .value("MyValue") + .build())).getManualAnnotationResponses() .get(0); assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); @@ -2789,12 +2819,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - ManualAddResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(), - file.getId(), - Set.of(ForceRedactionRequestModel.builder() - .annotationId("AnnotationId") - .legalBasis("legalBasis") - .build())).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(), + file.getId(), + Set.of(ForceRedactionRequestModel.builder() + .annotationId("AnnotationId") + .legalBasis("legalBasis") + .build())).getManualAnnotationResponses() .get(0); assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); @@ -2840,14 +2870,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - ManualAddResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(), - file.getId(), - Set.of(LegalBasisChangeRequestModel.builder() - .annotationId("AnnotationId") - .legalBasis("new legal basis") - .value("Luke Skywalker") - .section("") - .build())).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(), + file.getId(), + Set.of(LegalBasisChangeRequestModel.builder() + .annotationId("AnnotationId") + .legalBasis("new legal basis") + .value("Luke Skywalker") + .section("") + .build())).getManualAnnotationResponses() .get(0); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); @@ -2887,16 +2917,16 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - ManualAddResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(), - file.getId(), - Set.of(ResizeRedactionRequestModel.builder() - .annotationId("AnnotationId") - .updateDictionary(false) - .addToAllDossiers(false) - .value("Luke") - .positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1))) - .build()), - false).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(), + file.getId(), + Set.of(ResizeRedactionRequestModel.builder() + .annotationId("AnnotationId") + .updateDictionary(false) + .addToAllDossiers(false) + .value("Luke") + .positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1))) + .build()), + false).getManualAnnotationResponses() .get(0); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); @@ -2944,18 +2974,18 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - ManualAddResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), - file.getId(), - Set.of(RecategorizationRequestModel.builder() - .type(type2.getType()) - .annotationId("AnnotationId") - .addToDictionary(false) - .addToAllDossiers(false) - .legalBasis("") - .section("section") - .value("Luke Skywalker") - .build()), - false).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), + file.getId(), + Set.of(RecategorizationRequestModel.builder() + .type(type2.getType()) + .annotationId("AnnotationId") + .addToDictionary(false) + .addToAllDossiers(false) + .legalBasis("") + .section("section") + .value("Luke Skywalker") + .build()), + false).getManualAnnotationResponses() .get(0); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); @@ -2994,14 +3024,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - ManualAddResponse response = manualRedactionClient.removeRedactionBulk(dossier.getId(), - file.getId(), - Set.of(RemoveRedactionRequestModel.builder() - .annotationId("AnnotationId") - .removeFromDictionary(false) - .removeFromAllDossiers(false) - .build()), - false).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.removeRedactionBulk(dossier.getId(), + file.getId(), + Set.of(RemoveRedactionRequestModel.builder() + .annotationId("AnnotationId") + .removeFromDictionary(false) + .removeFromAllDossiers(false) + .build()), + false).getManualAnnotationResponses() .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); @@ -3047,18 +3077,18 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isEqualTo(true); - ManualAddResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), - file.getId(), - Set.of(RecategorizationRequestModel.builder() - .type(type2.getType()) - .annotationId("AnnotationId") - .addToDictionary(false) - .addToAllDossiers(false) - .legalBasis("Article legal basis") - .section("section") - .value("Luke Skywalker") - .build()), - false).getManualAddResponses() + ManualAnnotationResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), + file.getId(), + Set.of(RecategorizationRequestModel.builder() + .type(type2.getType()) + .annotationId("AnnotationId") + .addToDictionary(false) + .addToAllDossiers(false) + .legalBasis("Article legal basis") + .section("section") + .value("Luke Skywalker") + .build()), + false).getManualAnnotationResponses() .get(0); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); @@ -3105,7 +3135,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 600); } @@ -3137,7 +3167,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualRedactionResponse manualRedactionResponse = manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), forceRedactionRequestModels); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 600); } @@ -3179,7 +3209,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequestModels, false); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 600); } @@ -3216,12 +3246,57 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @Test - public void testBulkLocal() { + public void testBulkLocalAutomaticAnalysisOff() { var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); var file = fileTesterAndProvider.testAndProvideFile(dossier); + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), true); + testBulkLocal(dossierTemplate, dossier, file); + } + + + @Test + public void testBulkLocalAutomaticAnalysisOn() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), false); + testBulkLocal(dossierTemplate, dossier, file); + } + + + @Test + public void testRectangleRedactionsBulkLocalAutomaticAnalysisOff() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), true); + testRectangleRedactionsBulkLocal(dossierTemplate, dossier, file); + } + + + @Test + public void testRectangleRedactionsBulkLocalAutomaticAnalysisOn() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), false); + testRectangleRedactionsBulkLocal(dossierTemplate, dossier, file); + } + + + private void testBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) { + + whenGetEntityLogInvocation(); + var type1 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test1", false, 70); var type2 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test2", false, 71); var type3 = typeProvider.testAndProvideType(dossierTemplate, dossier, "test3", false, 72); @@ -3265,8 +3340,6 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); - when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); - String newLegal = "new Legal"; String otherSection = "other section"; ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), @@ -3279,12 +3352,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .value("Luke Skywalker37") .build(), false); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 1); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 1); + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(0).getEntityLogEntry().getType(), newType.getType()); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(0).getEntityLogEntry().getLegalBasis(), newLegal); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(0).getEntityLogEntry().getSection(), otherSection); manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), @@ -3298,12 +3371,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .originLegalBases(Set.of(legal3)) .build(), false); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 101); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 101); + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(1).getEntityLogEntry().getType(), newType.getType()); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(29).getEntityLogEntry().getLegalBasis(), newLegal); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(79).getEntityLogEntry().getSection(), otherSection); manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), @@ -3317,19 +3390,523 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .originTypes(Set.of(type2.getType())) .build(), false); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 101); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 101); + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(0).getEntityLogEntry().getType(), newType.getType()); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(36).getEntityLogEntry().getLegalBasis(), newLegal); - assertEquals(manualRedactionResponse.getManualAddResponses() + assertEquals(manualRedactionResponse.getManualAnnotationResponses() .get(98).getEntityLogEntry().getSection(), otherSection); manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).value(darthVader).build(), false); - assertEquals(manualRedactionResponse.getManualAddResponses().size(), 202); + assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 202); + } + + + private void testRectangleRedactionsBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) { + + whenGetEntityLogInvocation(); + + var typeManual = typeProvider.testAndProvideType(dossierTemplate, dossier, "manual", false, 70); + var newType = typeProvider.testAndProvideType(dossierTemplate, dossier, "newType", false, 71); + + String value = "non-readable content"; + String legalBasis = "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"; + + float x = 88.34f; + float y = 701.95f; + float height = 77.57000000000002f; + float width = 420.15f; + + List entityLogEntries = new ArrayList<>(); + + for (int page = 1; page <= 3; page++) { + Position position = new Position(new float[]{x, y, width, height}, page); + + EntityLogEntry entityLogEntry = EntityLogEntry.builder() + .id("RectangleAnnotation" + page) + .type(typeManual.getType()) + .value(value) + .entryType(EntryType.AREA) + .state(EntryState.APPLIED) + .legalBasis(legalBasis) + .positions(List.of(position)) + .engines(Set.of(Engine.MANUAL)) + .build(); + + entityLogEntries.add(entityLogEntry); + } + + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + Position filterPosition = new Position(new float[]{x, y, width, height}, 0); + + RecategorizationBulkLocalRequestModel recategorizationRequest = RecategorizationBulkLocalRequestModel.builder() + .rectangle(true) + .type(newType.getType()) + .legalBasis("new Legal Basis") + .section("new Section") + .position(filterPosition) + .build(); + + ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), recategorizationRequest, false); + + assertNotNull(manualRedactionResponse); + assertEquals(3, manualRedactionResponse.getManualAnnotationResponses().size()); + for (ManualAnnotationResponse response : manualRedactionResponse.getManualAnnotationResponses()) { + assertEquals(typeManual.getType(), response.getEntityLogEntry().getType()); + assertEquals("new Legal Basis", response.getEntityLogEntry().getLegalBasis()); + assertEquals("new Section", response.getEntityLogEntry().getSection()); + } + + recategorizationRequest = RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis("another Legal Basis") + .section("another Section") + .value(value) + .build(); + + RecategorizationBulkLocalRequestModel finalRecategorizationRequest = recategorizationRequest; + FeignException feignException = assertThrows(FeignException.class, + () -> manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), finalRecategorizationRequest, false)); + + assertEquals(feignException.status(), 400); + + recategorizationRequest = RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis("and another Legal Basis") + .section("and another Section") + .position(filterPosition) + .build(); + + feignException = assertThrows(FeignException.class, () -> manualRedactionClient.recategorizeBulkLocal(dossier.getId(), file.getId(), finalRecategorizationRequest, false)); + assertEquals(feignException.status(), 400); + + RemoveRedactionBulkLocalRequestModel removeRequest = RemoveRedactionBulkLocalRequestModel.builder().rectangle(true).position(filterPosition).build(); + + manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRequest, false); + + assertNotNull(manualRedactionResponse); + assertEquals(3, manualRedactionResponse.getManualAnnotationResponses().size()); + List removedEntries = entityLogService.getEntityLog(dossier.getId(), file.getId()).getEntityLogEntry() + .stream() + .filter(ele -> ele.getState().equals(EntryState.REMOVED)) + .toList(); + assertEquals(3, removedEntries.size()); + + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + removeRequest = RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).position(filterPosition).build(); + + RemoveRedactionBulkLocalRequestModel finalRemoveRequest = removeRequest; + assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), finalRemoveRequest, false)); + assertEquals(feignException.status(), 400); + } + + + @Test + public void testCaseSensitivityAutomaticAnalysisOff() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), true); + testCaseSensitivity(dossierTemplate, dossier, file); + } + + + @Test + public void testCaseSensitivityAutomaticAnalysisOn() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), false); + testCaseSensitivity(dossierTemplate, dossier, file); + } + + + private void testCaseSensitivity(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) { + + whenGetEntityLogInvocation(); + + var originalType = typeProvider.testAndProvideType(dossierTemplate, dossier, "originalType", false, 70); + var newType = typeProvider.testAndProvideType(dossierTemplate, dossier, "newType", false, 71); + + String valueLowerCase = "luke skywalker"; + String valueMixedCase = "Luke Skywalker"; + String valueUpperCase = "LUKE SKYWALKER"; + + List entityLogEntries = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationIdLowerCase" + i) + .type(originalType.getType()) + .value(valueLowerCase) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .section("Original Section") + .legalBasis("Original Legal Basis") + .engines(Set.of(Engine.MANUAL)) + .build()); + + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationIdMixedCase" + i) + .type(originalType.getType()) + .value(valueMixedCase) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .section("Original Section") + .legalBasis("Original Legal Basis") + .engines(Set.of(Engine.MANUAL)) + .build()); + + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationIdUpperCase" + i) + .type(originalType.getType()) + .value(valueUpperCase) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .section("Original Section") + .legalBasis("Original Legal Basis") + .engines(Set.of(Engine.MANUAL)) + .build()); + } + + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + String newLegalBasis = "New Legal Basis"; + String newSection = "New Section"; + + ManualRedactionResponse responseCaseSensitive = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), + file.getId(), + RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis(newLegalBasis) + .section(newSection) + .value(valueMixedCase) + .caseSensitive(true) + .build(), + false); + + assertEquals(10, responseCaseSensitive.getManualAnnotationResponses().size()); + for (ManualAnnotationResponse response : responseCaseSensitive.getManualAnnotationResponses()) { + assertEquals(newType.getType(), response.getEntityLogEntry().getType()); + assertEquals(valueMixedCase, response.getEntityLogEntry().getValue()); + assertEquals(newLegalBasis, response.getEntityLogEntry().getLegalBasis()); + assertEquals(newSection, response.getEntityLogEntry().getSection()); + } + + ManualRedactionResponse responseCaseInsensitive = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), + file.getId(), + RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis(newLegalBasis) + .section(newSection) + .value(valueMixedCase) + .caseSensitive(false) + .build(), + false); + + assertEquals(30, responseCaseInsensitive.getManualAnnotationResponses().size()); + for (ManualAnnotationResponse response : responseCaseInsensitive.getManualAnnotationResponses()) { + assertEquals(newType.getType(), response.getEntityLogEntry().getType()); + assertTrue(response.getEntityLogEntry().getValue().equalsIgnoreCase(valueMixedCase)); + assertEquals(newLegalBasis, response.getEntityLogEntry().getLegalBasis()); + assertEquals(newSection, response.getEntityLogEntry().getSection()); + } + + ManualRedactionResponse removalResponseCaseSensitive = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), + file.getId(), + RemoveRedactionBulkLocalRequestModel.builder() + .rectangle(false) + .value(valueMixedCase) + .caseSensitive(true) + .build(), + false); + + assertEquals(10, removalResponseCaseSensitive.getManualAnnotationResponses().size()); + + ManualRedactionResponse removalResponseCaseInsensitive = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), + file.getId(), + RemoveRedactionBulkLocalRequestModel.builder() + .rectangle(false) + .value(valueMixedCase) + .caseSensitive(false) + .build(), + false); + + assertEquals(30, removalResponseCaseInsensitive.getManualAnnotationResponses().size()); + } + + + @Test + public void testAddRemoveAddSameValueAndPositionAutomaticAnalysisOff() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + fileStatusPersistenceService.toggleAutomaticAnalysis(file.getId(), true); + testAddRemoveAddSameValueAndPosition(dossierTemplate, dossier, file); + } + + + private void testAddRemoveAddSameValueAndPosition(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) { + + whenGetEntityLogInvocation(); + + var typeOriginal = typeProvider.testAndProvideType(dossierTemplate, dossier, "originalType", false, 70); + + String value = "Test Value"; + String legalBasis = "Original Legal Basis"; + String section = "Original Section"; + + Position position = new Position(new float[]{100f, 200f, 50f, 50f}, 1); + + EntityLogEntry valueAnnotation = EntityLogEntry.builder() + .id("ValueAnnotation1") + .type(typeOriginal.getType()) + .value(value) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .section(section) + .legalBasis(legalBasis) + .engines(Set.of(Engine.MANUAL)) + .build(); + + EntityLogEntry rectangleAnnotation = EntityLogEntry.builder() + .id("RectangleAnnotation1") + .type(typeOriginal.getType()) + .value("non-readable content") + .entryType(EntryType.AREA) + .state(EntryState.APPLIED) + .positions(List.of(position)) + .legalBasis(legalBasis) + .section(section) + .engines(Set.of(Engine.MANUAL)) + .build(); + + List entityLogEntries = new ArrayList<>(); + entityLogEntries.add(valueAnnotation); + entityLogEntries.add(rectangleAnnotation); + + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + RemoveRedactionBulkLocalRequestModel removeValueRequest = RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).value(value).build(); + + ManualRedactionResponse removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, false); + + assertEquals(1, removeValueResponse.getManualAnnotationResponses().size()); + EntityLog currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(2, currentEntityLog.getEntityLogEntry().size()); + EntityLogEntry removedValueAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "ValueAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(removedValueAnnotation); + assertEquals(EntryState.REMOVED, removedValueAnnotation.getState()); + EntityLogEntry rectangleAnnotationEntry = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "RectangleAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(rectangleAnnotationEntry); + assertEquals(EntryState.APPLIED, rectangleAnnotationEntry.getState()); + + String awesomeComment = "awesome comment"; + BulkLocalResponse addValueBackResponse = BulkLocalResponse.builder() + .fileId(file.getId()) + .dossierId(dossier.getId()) + .type(typeOriginal.getType()) + .legalBasis(legalBasis) + .reason("Adding value annotation back") + .section(section) + .foundTerms(List.of(new FoundTerm(List.of(position), value))) + .comment(awesomeComment) + .build(); + searchTermOccurrencesResponseReceiver.receive(addValueBackResponse); + + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(3, currentEntityLog.getEntityLogEntry().size()); + removedValueAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "ValueAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(removedValueAnnotation); + assertEquals(EntryState.REMOVED, removedValueAnnotation.getState()); + EntityLogEntry newValueAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> !entry.getId().equals("ValueAnnotation1") && !entry.getId().equals("RectangleAnnotation1")) + .filter(entry -> entry.getEntryType() == EntryType.ENTITY) + .findFirst() + .orElse(null); + assertNotNull(newValueAnnotation); + assertEquals(EntryState.APPLIED, newValueAnnotation.getState()); + assertEquals(value, newValueAnnotation.getValue()); + List commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), newValueAnnotation.getId(), false); + assertEquals(commentEntities.size(), 1); + assertEquals(commentEntities.get(0).getText(), awesomeComment); + + removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, true); + + assertEquals(1, removeValueResponse.getManualAnnotationResponses().size()); + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(3, currentEntityLog.getEntityLogEntry().size()); + removedValueAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "ValueAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(removedValueAnnotation); + assertEquals(EntryState.REMOVED, removedValueAnnotation.getState()); + EntityLogEntry removedNewValueAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> !entry.getId().equals("ValueAnnotation1") && !entry.getId().equals("RectangleAnnotation1")) + .filter(entry -> entry.getEntryType() == EntryType.ENTITY) + .findFirst() + .orElse(null); + assertNotNull(removedNewValueAnnotation); + assertEquals(EntryState.REMOVED, removedNewValueAnnotation.getState()); + + rectangleAnnotationEntry = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "RectangleAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(rectangleAnnotationEntry); + assertEquals(EntryState.APPLIED, rectangleAnnotationEntry.getState()); + + searchTermOccurrencesResponseReceiver.receive(addValueBackResponse); + + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(4, currentEntityLog.getEntityLogEntry().size()); + List appliedValueAnnotations = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(value) && entry.getState() == EntryState.APPLIED) + .toList(); + assertEquals(1, appliedValueAnnotations.size()); + commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), appliedValueAnnotations.get(0).getId(), false); + assertEquals(commentEntities.size(), 1); + assertEquals(commentEntities.get(0).getText(), awesomeComment); + + String totallyDifferentComment = "totally different comment"; + RemoveRedactionBulkLocalRequestModel removeRectangleRequest = RemoveRedactionBulkLocalRequestModel.builder() + .rectangle(true) + .position(position) + .comment(totallyDifferentComment) + .build(); + + ManualRedactionResponse removeRectangleResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRectangleRequest, false); + + assertEquals(1, removeRectangleResponse.getManualAnnotationResponses().size()); + Long commentId = removeRectangleResponse.getManualAnnotationResponses() + .get(0).getCommentId(); + assertNotNull(commentId); + Optional comment = commentRepository.findById(commentId); + assertTrue(comment.isPresent()); + assertEquals(comment.get().getText(), totallyDifferentComment); + + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(4, currentEntityLog.getEntityLogEntry().size()); + rectangleAnnotationEntry = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "RectangleAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(rectangleAnnotationEntry); + assertEquals(EntryState.REMOVED, rectangleAnnotationEntry.getState()); + + AddRedactionRequestModel addRectangleRequest = AddRedactionRequestModel.builder() + .type(typeOriginal.getType()) + .positions(List.of(toRectangle(position))) + .legalBasis(legalBasis) + .section(section) + .reason("Adding rectangle annotation back") + .value("non-readable content") + .rectangle(true) + .build(); + ManualRedactionResponse addRectangleResponse = manualRedactionClient.addRedactionBulk(dossier.getId(), file.getId(), Set.of(addRectangleRequest)); + + assertEquals(1, addRectangleResponse.getManualAnnotationResponses().size()); + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(5, currentEntityLog.getEntityLogEntry().size()); + rectangleAnnotationEntry = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> "RectangleAnnotation1".equals(entry.getId())) + .findFirst() + .orElse(null); + assertNotNull(rectangleAnnotationEntry); + assertEquals(EntryState.REMOVED, rectangleAnnotationEntry.getState()); + EntityLogEntry newRectangleAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getEntryType() == EntryType.AREA && entry.getState() == EntryState.APPLIED && !entry.getId().equals("RectangleAnnotation1")) + .findFirst() + .orElse(null); + assertNotNull(newRectangleAnnotation); + + removeRectangleResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeRectangleRequest, true); + + assertEquals(1, removeRectangleResponse.getManualAnnotationResponses().size()); + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(5, currentEntityLog.getEntityLogEntry().size()); + EntityLogEntry removedNewRectangleAnnotation = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getEntryType() == EntryType.AREA && entry.getState() == EntryState.REMOVED && !entry.getId().equals("RectangleAnnotation1")) + .findFirst() + .orElse(null); + assertNotNull(removedNewRectangleAnnotation); + + addRectangleResponse = manualRedactionClient.addRedactionBulk(dossier.getId(), file.getId(), Set.of(addRectangleRequest)); + + assertEquals(1, addRectangleResponse.getManualAnnotationResponses().size()); + currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertEquals(6, currentEntityLog.getEntityLogEntry().size()); + long appliedRectangleAnnotationsCount = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getEntryType() == EntryType.AREA && entry.getState() == EntryState.APPLIED) + .count(); + assertEquals(1, appliedRectangleAnnotationsCount); + } + + + private Rectangle toRectangle(Position position) { + + return new Rectangle(position.x(), position.y(), position.w(), position.h(), 0); + } + + + private void whenGetEntityLogInvocation() { + + Answer answer = invocation -> { + + String dossierId = invocation.getArgument(0); + String fileId = invocation.getArgument(1); + + DossierEntity dossier = dossierService.getDossierById(dossierId); + EntityLog entityLog = fileManagementStorageService.getEntityLog(dossierId, fileId); + ManualRedactions unprocessedManualRedactions = manualRedactionProviderService.getManualRedactions(fileId, ManualChangesQueryOptions.unprocessedOnly()); + return entityLogMergeService.mergeEntityLog(unprocessedManualRedactions, entityLog, dossier); + }; + + when(entityLogService.getEntityLog(any(), any(), anyList(), anyBoolean())).thenAnswer(answer); + when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenAnswer(answer); + when(entityLogService.getEntityLog(any(), any())).thenAnswer(answer); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java index 789e8440e..1b4e793cf 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java @@ -1,70 +1,32 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.mock.web.MockMultipartFile; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient; -import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; -import com.iqser.red.service.peristence.v1.server.integration.client.ReportTemplateClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob; -import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadCompressionMessageReceiver; -import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver; -import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver; -import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.PendingEntryFactory; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository; import com.iqser.red.service.persistence.management.v1.processor.service.queue.SearchTermOccurrencesResponseReceiver; import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse; -import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; -import com.iqser.red.service.persistence.service.v1.api.shared.model.EntityPosition; -import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.PrepareDownloadWithOptionRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FoundTerm; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.ReportTemplate; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel; -import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; -import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; -import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.tenantcommons.TenantContext; -import com.knecon.fforesight.tenantcommons.model.TenantResponse; import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) @@ -101,12 +63,12 @@ public class SearchTermOccurrencesTest extends AbstractPersistenceServerServiceT searchTermOccurrencesResponseReceiver.receive(BulkLocalResponse.builder() .fileId(file.getId()) .dossierId(dossier.getId()) - .searchTerm("searchTerm") .type(type.getType()) .legalBasis("legalBasis") .reason("reason") .section("section") - .entityPositions(List.of(new EntityPosition(List.of(new Position(new float[]{1f, 2f, 3f, 4f}, 1))))) + .foundTerms(List.of(new FoundTerm(List.of(new Position(new float[]{1f, 2f, 3f, 4f}, 1)), "searchTerm"))) + .comment(null) .build()); List newEntries = manualRedactionRepository.findByFileIdAndOptions(file.getId(), false, false, false); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/local-bulk-test.ENTITY_LOG.json b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/local-bulk-test.ENTITY_LOG.json index e717a1dcd..27d73e8b1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/local-bulk-test.ENTITY_LOG.json +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/local-bulk-test.ENTITY_LOG.json @@ -2024,6 +2024,243 @@ "importedRedactionIntersections": [], "numberOfComments": 0 }, + { + "id": "e5a6654664d689949e3823e1681d6272123", + "type": "CBI_author", + "entryType": "AREA", + "state": "APPLIED", + "value": "Akesson", + "reason": "Author found, removed by manual override, forced by manual override", + "matchedRule": "CBI.0.0", + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "imported": false, + "containingNodeId": [ + 0, + 0 + ], + "closestHeadline": "", + "section": "[0, 0]: Paragraph: $ D Jones $.Friedrich", + "color": null, + "positions": [ + { + "rectangle": [ + 322.63608, + 308.1, + 41.375977, + 12.642 + ], + "pageNumber": 1 + } + ], + "textBefore": "Akashi Akashiba Akerman ", + "textAfter": " Akhavan Akhavan M", + "startOffset": 2806, + "endOffset": 2813, + "imageHasTransparency": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false, + "excluded": false, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2024-03-13T08:44:54.811245839Z" + }, + { + "analysisNumber": 2, + "type": "REMOVED", + "dateTime": "2024-03-13T08:45:53.93986414Z" + }, + { + "analysisNumber": 7, + "type": "ADDED", + "dateTime": "2024-03-13T08:59:31.444615299Z" + } + ], + "manualChanges": [ + { + "manualRedactionType": "REMOVE", + "processedDate": "2024-03-13T09:05:21.954045754Z", + "requestedDate": "2024-03-13T08:45:49.298559Z", + "userId": "abbc3e3a-c499-48da-9346-063e1863a7f5", + "propertyChanges": {} + }, + { + "manualRedactionType": "FORCE", + "processedDate": "2024-03-13T09:05:21.954046415Z", + "requestedDate": "2024-03-13T08:59:29.8778Z", + "userId": "abbc3e3a-c499-48da-9346-063e1863a7f5", + "propertyChanges": { + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002" + } + } + ], + "engines": [ + "MANUAL", + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "numberOfComments": 0 + }, + { + "id": "e5a6654664d689949e3823e1681d6273123", + "type": "CBI_author", + "entryType": "AREA", + "state": "APPLIED", + "value": "Akesson", + "reason": "Author found, removed by manual override, forced by manual override", + "matchedRule": "CBI.0.0", + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "imported": false, + "containingNodeId": [ + 0, + 0 + ], + "closestHeadline": "", + "section": "[0, 0]: Paragraph: $ D Jones $.Friedrich", + "color": null, + "positions": [ + { + "rectangle": [ + 322.63608, + 308.1, + 41.375977, + 12.642 + ], + "pageNumber": 2 + } + ], + "textBefore": "Akashi Akashiba Akerman ", + "textAfter": " Akhavan Akhavan M", + "startOffset": 2806, + "endOffset": 2813, + "imageHasTransparency": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false, + "excluded": false, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2024-03-13T08:44:54.811245839Z" + }, + { + "analysisNumber": 2, + "type": "REMOVED", + "dateTime": "2024-03-13T08:45:53.93986414Z" + }, + { + "analysisNumber": 7, + "type": "ADDED", + "dateTime": "2024-03-13T08:59:31.444615299Z" + } + ], + "manualChanges": [ + { + "manualRedactionType": "REMOVE", + "processedDate": "2024-03-13T09:05:21.954045754Z", + "requestedDate": "2024-03-13T08:45:49.298559Z", + "userId": "abbc3e3a-c499-48da-9346-063e1863a7f5", + "propertyChanges": {} + }, + { + "manualRedactionType": "FORCE", + "processedDate": "2024-03-13T09:05:21.954046415Z", + "requestedDate": "2024-03-13T08:59:29.8778Z", + "userId": "abbc3e3a-c499-48da-9346-063e1863a7f5", + "propertyChanges": { + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002" + } + } + ], + "engines": [ + "MANUAL", + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "numberOfComments": 0 + }, + { + "id": "e5a6654664d689949e3823e1681d6274123", + "type": "CBI_author", + "entryType": "AREA", + "state": "APPLIED", + "value": "Akesson", + "reason": "Author found, removed by manual override, forced by manual override", + "matchedRule": "CBI.0.0", + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "imported": false, + "containingNodeId": [ + 0, + 0 + ], + "closestHeadline": "", + "section": "[0, 0]: Paragraph: $ D Jones $.Friedrich", + "color": null, + "positions": [ + { + "rectangle": [ + 322.63608, + 308.1, + 41.375977, + 12.642 + ], + "pageNumber": 3 + } + ], + "textBefore": "Akashi Akashiba Akerman ", + "textAfter": " Akhavan Akhavan M", + "startOffset": 2806, + "endOffset": 2813, + "imageHasTransparency": false, + "dictionaryEntry": true, + "dossierDictionaryEntry": false, + "excluded": false, + "changes": [ + { + "analysisNumber": 1, + "type": "ADDED", + "dateTime": "2024-03-13T08:44:54.811245839Z" + }, + { + "analysisNumber": 2, + "type": "REMOVED", + "dateTime": "2024-03-13T08:45:53.93986414Z" + }, + { + "analysisNumber": 7, + "type": "ADDED", + "dateTime": "2024-03-13T08:59:31.444615299Z" + } + ], + "manualChanges": [ + { + "manualRedactionType": "REMOVE", + "processedDate": "2024-03-13T09:05:21.954045754Z", + "requestedDate": "2024-03-13T08:45:49.298559Z", + "userId": "abbc3e3a-c499-48da-9346-063e1863a7f5", + "propertyChanges": {} + }, + { + "manualRedactionType": "FORCE", + "processedDate": "2024-03-13T09:05:21.954046415Z", + "requestedDate": "2024-03-13T08:59:29.8778Z", + "userId": "abbc3e3a-c499-48da-9346-063e1863a7f5", + "propertyChanges": { + "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002" + } + } + ], + "engines": [ + "MANUAL", + "DICTIONARY" + ], + "reference": [], + "importedRedactionIntersections": [], + "numberOfComments": 0 + }, { "id": "ad3096cde99eac5a9c41ec85058efcfe", "type": "CBI_author", diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java index 6f88f645c..59fdcd412 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java @@ -34,4 +34,6 @@ public class BulkLocalRequest { @Builder.Default private Set pageNumbers = Collections.emptySet(); + private String comment; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java index 725b1ef49..3fa1a460d 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java @@ -23,9 +23,6 @@ public class BulkLocalResponse { @NonNull private String fileId; - @NonNull - private String searchTerm; - @NonNull private String type; @@ -38,7 +35,8 @@ public class BulkLocalResponse { @NonNull @Builder.Default - private List entityPositions = new ArrayList<>(); + private List foundTerms = new ArrayList<>(); + private String comment; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/EntityPosition.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FoundTerm.java similarity index 75% rename from persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/EntityPosition.java rename to persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FoundTerm.java index 3001a7e25..98e51efda 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/EntityPosition.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FoundTerm.java @@ -4,6 +4,6 @@ import java.util.List; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -public record EntityPosition(List positions) { +public record FoundTerm(List positions, String value) { } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAnnotationResponse.java similarity index 92% rename from persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java rename to persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAnnotationResponse.java index 594f2f5d8..178691fb9 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAddResponse.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualAnnotationResponse.java @@ -11,7 +11,7 @@ import lombok.NoArgsConstructor; @Builder @AllArgsConstructor @NoArgsConstructor -public class ManualAddResponse { +public class ManualAnnotationResponse { private String annotationId; private Long commentId; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRedactionResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRedactionResponse.java index 9f56198db..cc1995bc1 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRedactionResponse.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRedactionResponse.java @@ -13,5 +13,5 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class ManualRedactionResponse { - private List manualAddResponses; + private List manualAnnotationResponses; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java index eed6578e3..7b4d8a0d0 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java @@ -42,4 +42,6 @@ public class AddRedactionBulkLocalRequestModel { @Builder.Default private Set pageNumbers = Collections.emptySet(); + private String comment; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RecategorizationBulkLocalRequestModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RecategorizationBulkLocalRequestModel.java index b3b44fb06..546453c48 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RecategorizationBulkLocalRequestModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RecategorizationBulkLocalRequestModel.java @@ -39,4 +39,6 @@ public class RecategorizationBulkLocalRequestModel { @Builder.Default private Set pageNumbers = Collections.emptySet(); + private String comment; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionBulkLocalRequestModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionBulkLocalRequestModel.java index 65caf655d..0785afeeb 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionBulkLocalRequestModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/RemoveRedactionBulkLocalRequestModel.java @@ -36,4 +36,6 @@ public class RemoveRedactionBulkLocalRequestModel { @Builder.Default private Set pageNumbers = Collections.emptySet(); + private String comment; + } diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepository.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepository.java index 1526055e0..63b0a4ca5 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepository.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepository.java @@ -3,11 +3,18 @@ package com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository import java.util.List; import java.util.Set; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument; + public interface EntityLogEntryDocumentCustomRepository { - List findEntryIdsByValueWithFilters(String value, boolean caseSensitive, Set originTypes, Set originLegalBases, Set pageNumbers); + List findEntryIdsByValueWithFilters(String entityLogId, + String value, + boolean caseSensitive, + Set originTypes, + Set originLegalBases, + Set pageNumbers); - List findEntryIdsByMatchingPositionWithFilters(float[] rectangle, Set originTypes, Set originLegalBases, Set pageNumbers); + List findEntryIdsByMatchingPositionWithFilters(String entityLogId, float[] rectangle, Set originTypes, Set originLegalBases, Set pageNumbers); } diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepositoryImpl.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepositoryImpl.java index 42124dc5d..9dbd41eab 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepositoryImpl.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentCustomRepositoryImpl.java @@ -4,13 +4,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Repository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument; import jakarta.validation.constraints.NotNull; @@ -27,10 +27,16 @@ public class EntityLogEntryDocumentCustomRepositoryImpl implements EntityLogEntr @Override - public List findEntryIdsByValueWithFilters(String value, boolean caseSensitive, Set originTypes, Set originLegalBases, Set pageNumbers) { + public List findEntryIdsByValueWithFilters(String entityLogId, + String value, + boolean caseSensitive, + Set originTypes, + Set originLegalBases, + Set pageNumbers) { Query query = new Query(); List criteriaList = new ArrayList<>(); + criteriaList.add(Criteria.where("entryType").ne(EntryType.AREA)); if (caseSensitive) { criteriaList.add(Criteria.where("value").is(value)); @@ -38,34 +44,41 @@ public class EntityLogEntryDocumentCustomRepositoryImpl implements EntityLogEntr criteriaList.add(Criteria.where("value").regex(Pattern.compile("^" + Pattern.quote(value) + "$", Pattern.CASE_INSENSITIVE))); } - addCommonCriteria(originTypes, originLegalBases, pageNumbers, criteriaList); + addCommonCriteria(entityLogId, originTypes, originLegalBases, pageNumbers, criteriaList); Criteria mergedCriteria = new Criteria().andOperator(criteriaList.toArray(new Criteria[0])); query.addCriteria(mergedCriteria); - return executeQuery(query); + return mongoTemplate.find(query, EntityLogEntryDocument.class); } @Override - public List findEntryIdsByMatchingPositionWithFilters(float[] rectangle, Set originTypes, Set originLegalBases, Set pageNumbers) { + public List findEntryIdsByMatchingPositionWithFilters(String entityLogId, + float[] rectangle, + Set originTypes, + Set originLegalBases, + Set pageNumbers) { Query query = new Query(); List criteriaList = new ArrayList<>(); + criteriaList.add(Criteria.where("entryType").is(EntryType.AREA)); criteriaList.add(Criteria.where("positions").elemMatch(Criteria.where("rectangle").is(rectangle))); - addCommonCriteria(originTypes, originLegalBases, pageNumbers, criteriaList); + addCommonCriteria(entityLogId, originTypes, originLegalBases, pageNumbers, criteriaList); Criteria mergedCriteria = new Criteria().andOperator(criteriaList.toArray(new Criteria[0])); query.addCriteria(mergedCriteria); - return executeQuery(query); + return mongoTemplate.find(query, EntityLogEntryDocument.class); } @NotNull - private void addCommonCriteria(Set originTypes, Set originLegalBases, Set pageNumbers, List criteriaList) { + private void addCommonCriteria(String entityLogId, Set originTypes, Set originLegalBases, Set pageNumbers, List criteriaList) { + + criteriaList.add(Criteria.where("entityLogId").is(entityLogId)); if (originTypes != null && !originTypes.isEmpty()) { criteriaList.add(Criteria.where("type").in(originTypes)); @@ -82,14 +95,5 @@ public class EntityLogEntryDocumentCustomRepositoryImpl implements EntityLogEntr } - private List executeQuery(Query query) { - - query.fields().include("entryId").exclude("id"); - - return mongoTemplate.find(query, EntityLogEntryDocument.class) - .stream() - .map(EntityLogEntryDocument::getEntryId) - .collect(Collectors.toList()); - } } diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java index 5d758659b..b29f699cc 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java @@ -338,27 +338,68 @@ public class EntityLogMongoService { } - public Set findEntryIdsOfManualsWithValue(String value) { + public Set findEntriesByValueWithFilters(String dossierId, + String fileId, + String value, + boolean caseSensitive, + Set originTypes, + Set originLegalBases, + Set pageNumbers) { - return new HashSet<>(entityLogEntryDocumentRepository.findEntryIdsByValueAndEngineManual(value)); + return new HashSet<>(entityLogEntryDocumentCustomRepository.findEntryIdsByValueWithFilters(mapper.getLogId(dossierId, fileId), + value, + caseSensitive, + originTypes, + originLegalBases, + pageNumbers) + .stream() + .map(mapper::fromLogEntryDocument) + .toList()); } - public Set findEntryIdsOfManualsWithPositionRectangle(float[] rectangle) { + public Set findEntriesByMatchingFullPositionWithFilters(String dossierId, + String fileId, + float[] rectangle, + Set originTypes, + Set originLegalBases, + Set pageNumbers) { - return new HashSet<>(entityLogEntryDocumentRepository.findEntryIdsByMatchingPositionAndEngineManual(rectangle)); + return new HashSet<>(entityLogEntryDocumentCustomRepository.findEntryIdsByMatchingPositionWithFilters(mapper.getLogId(dossierId, fileId), + rectangle, + originTypes, + originLegalBases, + pageNumbers) + .stream() + .map(mapper::fromLogEntryDocument) + .toList()); } - public Set findEntryIdsByValueWithFilters(String value, boolean caseSensitive, Set originTypes, Set originLegalBases, Set pageNumbers) { + public Set findEntryIdsByMatchingFullPositionWithFilters(String dossierId, + String fileId, + float[] rectangle, + Set originTypes, + Set originalLegalBases, + Set pageNumbers) { - return new HashSet<>(entityLogEntryDocumentCustomRepository.findEntryIdsByValueWithFilters(value, caseSensitive, originTypes, originLegalBases, pageNumbers)); + return findEntriesByMatchingFullPositionWithFilters(dossierId, fileId, rectangle, originTypes, originalLegalBases, pageNumbers).stream() + .map(EntityLogEntry::getId) + .collect(Collectors.toSet()); } - public Set findEntryIdsByMatchingFullPositionWithFilters(float[] rectangle, Set originTypes, Set originLegalBases, Set pageNumbers) { + public Set findEntryIdsByValueWithFilters(String dossierId, + String fileId, + String value, + boolean caseSensitive, + Set originTypes, + Set originLegalBases, + Set pageNumbers) { - return new HashSet<>(entityLogEntryDocumentCustomRepository.findEntryIdsByMatchingPositionWithFilters(rectangle, originTypes, originLegalBases, pageNumbers)); + return findEntriesByValueWithFilters(dossierId, fileId, value, caseSensitive, originTypes, originLegalBases, pageNumbers).stream() + .map(EntityLogEntry::getId) + .collect(Collectors.toSet()); } }