From 14da82e2b7e39bb1310572ba69dee3cd129c6b09 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Tue, 16 Jul 2024 14:01:44 +0300 Subject: [PATCH] RED-9606 - Bulk actions on many annotations --- .../ManualRedactionMapper.java | 13 +- .../ManualRedactionService.java | 18 +- .../tests/ManualRedactionTest.java | 162 ++++++++++++++++-- 3 files changed, 170 insertions(+), 23 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index 7c15e8c26..4bb71e12f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -204,11 +204,15 @@ public class ManualRedactionMapper { .toList(), includeUnprocessed); + Map annotationIdToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); List> requests = new ArrayList<>(); for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) { - entityLogEntries.forEach(entityLogEntry -> { + EntityLogEntry entityLogEntry = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId()); + + if (entityLogEntry != null) { String changedValue; String changedTypeId; String uuid = UUID.randomUUID().toString(); @@ -255,7 +259,7 @@ public class ManualRedactionMapper { } requests.add(RequestEntryPair.builder().request(request).entityLogEntry(entityLogEntry).build()); - }); + } } return requests; @@ -319,10 +323,7 @@ public class ManualRedactionMapper { .addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers()) .build(); - if (!entityLogEntry.getEngines().contains(Engine.MANUAL) - && !request.isAddToAllDossiers() - && !request.getUpdateDictionary() - && entryIsEntityType(entityLogEntry)) { + if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary() && entryIsEntityType(entityLogEntry)) { request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId()); entityLogEntry.setId(uuid); manualRedactionEntryConsumer.accept(entityLogEntry); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java index 1ef9ec401..14aa95aad 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java @@ -6,6 +6,7 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -197,6 +198,12 @@ public class ManualRedactionService { String dossierTemplateId, boolean includeUnprocessed) { + var numberOfDictRemoves = removeRedactionRequests.stream() + .filter(removeRedactionRequestModel -> removeRedactionRequestModel.isRemoveFromDictionary() || removeRedactionRequestModel.isRemoveFromAllDossiers()) + .count(); + if (numberOfDictRemoves > 100) { + throw new BadRequestException("Maximum number of remove from dictionary requests is 100."); + } var response = new ArrayList(); List> requests = manualRedactionMapper.toRemoveRedactionRequestList(dossierId, fileId, @@ -204,13 +211,18 @@ public class ManualRedactionService { removeRedactionRequests, includeUnprocessed); + Set verifiedTypes = new HashSet<>(); //validate removing from dossier template dictionary for (RequestEntryPair removeRedactionRequestRequestEntryPair : requests) { RemoveRedactionRequest removeRedactionRequest = removeRedactionRequestRequestEntryPair.getRequest(); EntityLogEntry entityLogEntry = removeRedactionRequestRequestEntryPair.getEntityLogEntry(); - manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest, - removeRedactionRequest.getTypeToRemove(), - removeRedactionRequest.getDossierTemplateId()); + + if (verifiedTypes.add(removeRedactionRequest.getTypeToRemove())) { + manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest, + removeRedactionRequest.getTypeToRemove(), + removeRedactionRequest.getDossierTemplateId()); + } + log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId()); IdRemovalEntity idRemoval = removeRedactionPersistenceService.insert(fileId, removeRedactionRequest); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index c42923419..d74217018 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1847,20 +1847,29 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), - Set.of(RecategorizationRequestModel.builder().annotationId("dv").legalBasis("").section("section").type(type.getType()).build()), + Set.of(RecategorizationRequestModel.builder() + .annotationId("dv") + .legalBasis("") + .section("section") + .type(type.getType()) + .build()), false); var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); assertEquals(allManualRedactions.getRecategorizations().size(), 1); assertTrue(allManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response.getManualAddResponses() + .get(0).getAnnotationId()))); var unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertEquals(unprocessedManualRedactions.getRecategorizations().size(), 1); assertTrue(unprocessedManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response.getManualAddResponses() + .get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), @@ -1873,27 +1882,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .build()); ManualRedactionResponse response2 = manualRedactionClient.recategorizeBulk(dossier.getId(), - file.getId(), - Set.of(RecategorizationRequestModel.builder().annotationId("dv2").legalBasis("").section("section").type(type.getType()).build()), - false); + file.getId(), + Set.of(RecategorizationRequestModel.builder() + .annotationId("dv2") + .legalBasis("") + .section("section") + .type(type.getType()) + .build()), + false); allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); assertEquals(allManualRedactions.getRecategorizations().size(), 2); assertTrue(allManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response.getManualAddResponses() + .get(0).getAnnotationId()))); assertTrue(allManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response2.getManualAddResponses() + .get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertEquals(unprocessedManualRedactions.getRecategorizations().size(), 1); assertTrue(unprocessedManualRedactions.getRecategorizations() .stream() - .noneMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId()))); + .noneMatch(entry -> entry.getAnnotationId() + .equals(response.getManualAddResponses() + .get(0).getAnnotationId()))); assertTrue(unprocessedManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response2.getManualAddResponses() + .get(0).getAnnotationId()))); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), @@ -1909,10 +1931,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertEquals(allManualRedactions.getRecategorizations().size(), 2); assertTrue(allManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response.getManualAddResponses() + .get(0).getAnnotationId()))); assertTrue(allManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response2.getManualAddResponses() + .get(0).getAnnotationId()))); unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true); assertTrue(unprocessedManualRedactions.getRecategorizations().isEmpty()); @@ -2249,13 +2275,15 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .section("overriddenSection") .build(); - ManualRedactionResponse response =manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false); + ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false); var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true); assertEquals(1, allManualRedactions.getRecategorizations().size()); assertTrue(allManualRedactions.getRecategorizations() .stream() - .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId()))); + .anyMatch(entry -> entry.getAnnotationId() + .equals(response.getManualAddResponses() + .get(0).getAnnotationId()))); assertTrue(allManualRedactions.getRecategorizations() .stream() .anyMatch(entry -> entry.getLegalBasis().equals("lb2"))); @@ -3028,4 +3056,110 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); } + + @Test + public void testForceMoreThan500Annotations() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + List entityLogEntries = new ArrayList<>(); + Set forceRedactionRequestModels = new HashSet<>(); + for (int i = 0; i < 600; i++) { + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId" + i) + .type("test") + .value("Luke Skywalker" + i) + .entryType(EntryType.ENTITY) + .state(EntryState.SKIPPED) + .dictionaryEntry(false) + .section("section") + .legalBasis("Article legal basis") + .build()); + forceRedactionRequestModels.add(ForceRedactionRequestModel.builder().annotationId("AnnotationId" + i).legalBasis("lb1").comment("").build()); + } + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); + + ManualRedactionResponse manualRedactionResponse = manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), forceRedactionRequestModels); + assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + } + + + @Test + public void testRecategorizeMoreThan500Annotations() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate, null, "type", true); + assertThat(type.isDossierDictionaryOnly()).isTrue(); + + List entityLogEntries = new ArrayList<>(); + Set recategorizationRequestModels = new HashSet<>(); + for (int i = 0; i < 600; i++) { + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId" + i) + .type("test") + .value("Luke Skywalker" + i) + .entryType(EntryType.ENTITY) + .state(EntryState.SKIPPED) + .dictionaryEntry(false) + .section("section") + .legalBasis("Article legal basis") + .build()); + recategorizationRequestModels.add(RecategorizationRequestModel.builder() + .annotationId("AnnotationId" + i) + .legalBasis("lb1") + .type("type") + .addToAllDossiers(false) + .addToDictionary(false) + .comment("") + .build()); + } + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); + + ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequestModels, false); + assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + } + + + @Test + public void testRemoveFromDictMoreThan100Annotations() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + List entityLogEntries = new ArrayList<>(); + Set removeRedactionRequestModels = new HashSet<>(); + for (int i = 0; i < 101; i++) { + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId" + i) + .type("test") + .value("Luke Skywalker" + i) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .dictionaryEntry(true) + .section("section") + .legalBasis("Article legal basis") + .build()); + removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder().annotationId("AnnotationId" + i).removeFromDictionary(true).removeFromAllDossiers(true).build()); + } + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); + + var result = assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false)); + assertTrue(result.getMessage().contains("Maximum number of remove from dictionary requests is 100.")); + } + } -- 2.47.2