From 73ce9cec827ce95a00d9374bc4674df736a64f9e Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Thu, 11 Jul 2024 15:03:02 +0300 Subject: [PATCH] RED-9606 - Fix removal --- .../ManualRedactionMapper.java | 13 +- .../tests/ManualRedactionTest.java | 209 +++++++++++------- 2 files changed, 145 insertions(+), 77 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index a37dd0bba..71cf050f8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -4,9 +4,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -91,10 +94,16 @@ public class ManualRedactionMapper { .map(RemoveRedactionRequestModel::getAnnotationId) .toList(), includeUnprocessed); + + Map iddToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + List> requests = new ArrayList<>(); for (var removeRedactionRequest : removeRedactionRequests) { - entityLogEntries.forEach(entityLogEntry -> { + EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId()); + + if (entityLogEntry != null) { var request = RemoveRedactionRequest.builder() .annotationId(removeRedactionRequest.getAnnotationId()) .user(KeycloakSecurity.getUserId()) @@ -111,7 +120,7 @@ public class ManualRedactionMapper { } requests.add(RequestEntryPair.builder().request(request.build()).entityLogEntry(entityLogEntry).build()); - }); + } } return requests; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 3de46caef..9f11c7e17 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.when; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.Dictionary; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -411,8 +413,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); @@ -470,8 +472,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .removeFromDictionary(true) .removeFromAllDossiers(true) .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); @@ -534,7 +536,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier") .dossierDictionaryEntry(true) @@ -542,7 +545,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -561,7 +565,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -579,7 +584,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier 1 var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .comment("resized dossier redaction") .value("test redaction in dossier dictionary") .updateDictionary(true) @@ -699,7 +705,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier yayy") .dossierDictionaryEntry(true) @@ -707,7 +714,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -726,7 +734,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -744,7 +753,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier 1 var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .comment("resized dossier redaction") .value("test redaction in dossier") .updateDictionary(true) @@ -868,7 +878,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier") .dossierDictionaryEntry(true) @@ -876,7 +887,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -895,7 +907,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -913,7 +926,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier dict var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .comment("resized dossier template redaction") .value("test redaction in dossier template dictionary") .updateDictionary(true) @@ -1034,7 +1048,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier yayy") .dossierDictionaryEntry(true) @@ -1042,7 +1057,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -1061,7 +1077,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -1079,7 +1096,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier dict var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .comment("resized dossier template redaction") .value("test redaction in dossier template") .updateDictionary(true) @@ -2038,8 +2056,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .stream() .anyMatch(entry -> entry.getAnnotationId().equals("annotationId"))); - assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false) - .getManualAddResponses().get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long"); + assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses() + .get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long"); assertNull(allManualRedactions.getRecategorizations() .stream() @@ -2503,7 +2521,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .entryType(EntryType.ENTITY) .state(EntryState.APPLIED) .dictionaryEntry(true) - .positions(List.of(new Position(10f, 10f ,10f, 10f, 1))) + .positions(List.of(new Position(10f, 10f, 10f, 10f, 1))) .build()), null, 0, @@ -2514,22 +2532,22 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(), - file.getId(), - Set.of(AddRedactionRequestModel.builder() - .sourceId("") - .reason("Reason") - .type(type.getType()) - .addToAllDossiers(false) - .addToDictionary(false) - .dictionaryEntryType(DictionaryEntryType.ENTRY) - .forceAddToDictionary(false) - .legalBasis("legalBasis") - .positions(List.of(new Rectangle(10f, 10f ,10f, 10f, 1))) - .value("MyValue") - .build())) - .getManualAddResponses().get(0); + file.getId(), + Set.of(AddRedactionRequestModel.builder() + .sourceId("") + .reason("Reason") + .type(type.getType()) + .addToAllDossiers(false) + .addToDictionary(false) + .dictionaryEntryType(DictionaryEntryType.ENTRY) + .forceAddToDictionary(false) + .legalBasis("legalBasis") + .positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1))) + .value("MyValue") + .build())).getManualAddResponses() + .get(0); - assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); + assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); assertEquals(response.getEntityLogEntry().getValue(), "MyValue"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis"); assertEquals(response.getEntityLogEntry().getReason(), "Reason"); @@ -2560,7 +2578,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .entryType(EntryType.ENTITY) .state(EntryState.SKIPPED) .dictionaryEntry(true) - .positions(List.of(new Position(10f, 10f ,10f, 10f, 1))) + .positions(List.of(new Position(10f, 10f, 10f, 10f, 1))) .build()), null, 0, @@ -2571,14 +2589,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(), - file.getId(), - Set.of(ForceRedactionRequestModel.builder() - .annotationId("AnnotationId") - .legalBasis("legalBasis") - .build())) - .getManualAddResponses().get(0); + file.getId(), + Set.of(ForceRedactionRequestModel.builder() + .annotationId("AnnotationId") + .legalBasis("legalBasis") + .build())).getManualAddResponses() + .get(0); - assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); + assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis"); assertEquals(response.getEntityLogEntry().getState(), EntryState.APPLIED); assertEquals(response.getEntityLogEntry().getManualChanges() @@ -2616,14 +2634,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(), - file.getId(), - Set.of(LegalBasisChangeRequestModel.builder() - .annotationId("AnnotationId") - .legalBasis("new legal basis") - .value("Luke Skywalker") - .section("") - .build())) - .getManualAddResponses().get(0); + file.getId(), + Set.of(LegalBasisChangeRequestModel.builder() + .annotationId("AnnotationId") + .legalBasis("new legal basis") + .value("Luke Skywalker") + .section("") + .build())).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis"); @@ -2633,6 +2651,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .get(0).getProcessedDate()); } + @Test public void testLocalResize() { @@ -2651,7 +2670,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .entryType(EntryType.ENTITY) .state(EntryState.APPLIED) .dictionaryEntry(true) - .positions(List.of(new Position(10f, 10f ,10f, 10f, 1))) + .positions(List.of(new Position(10f, 10f, 10f, 10f, 1))) .build()), null, 0, @@ -2662,23 +2681,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(), - file.getId(), - Set.of(ResizeRedactionRequestModel.builder() - .annotationId("AnnotationId") - .updateDictionary(false) - .addToAllDossiers(false) - .value("Luke") - .positions(List.of(new Rectangle(5f, 5f, 5f ,5f ,1))) - .build()), - false) - .getManualAddResponses().get(0); + file.getId(), + Set.of(ResizeRedactionRequestModel.builder() + .annotationId("AnnotationId") + .updateDictionary(false) + .addToAllDossiers(false) + .value("Luke") + .positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1))) + .build()), + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getValue(), "Luke"); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[0], 5f); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[1], 5f); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[2], 5f); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[3], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[0], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[1], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[2], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[3], 5f); assertEquals(response.getEntityLogEntry().getManualChanges() .get(0).getManualRedactionType(), ManualRedactionType.RESIZE); assertNull(response.getEntityLogEntry().getManualChanges() @@ -2725,8 +2748,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .section("section") .value("Luke Skywalker") .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getType(), type2.getType()); @@ -2771,8 +2794,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .removeFromDictionary(false) .removeFromAllDossiers(false) .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED); @@ -2828,8 +2851,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .section("section") .value("Luke Skywalker") .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getType(), type2.getType()); @@ -2842,4 +2865,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .get(0).getProcessedDate()); } + + @Test + public void testRemoveMoreThan500Annotations() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + List entityLogEntries = new ArrayList<>(); + Set removeRedactionRequestModels = new HashSet<>(); + for (int i = 0; i < 600; i++) { + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId" + i) + .type("test") + .value("Luke Skywalker" + i) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .dictionaryEntry(false) + .section("section") + .legalBasis("Article legal basis") + .build()); + removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder() + .annotationId("AnnotationId" + i) + .removeFromDictionary(false) + .removeFromAllDossiers(false) + .build()); + } + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); + + ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false); + assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + } + } -- 2.47.2