From 0f3434bca15e0a96acadb350a6f2317f2e29e372 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Thu, 11 Jul 2024 15:11:52 +0300 Subject: [PATCH] RED-9606 - Fix removal --- .../ManualRedactionMapper.java | 13 +- .../tests/ManualRedactionTest.java | 209 +++++++++++------- 2 files changed, 145 insertions(+), 77 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java index a37dd0bba..71cf050f8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java @@ -4,9 +4,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -91,10 +94,16 @@ public class ManualRedactionMapper { .map(RemoveRedactionRequestModel::getAnnotationId) .toList(), includeUnprocessed); + + Map iddToEntityLogEntryMap = entityLogEntries.stream() + .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity())); + List> requests = new ArrayList<>(); for (var removeRedactionRequest : removeRedactionRequests) { - entityLogEntries.forEach(entityLogEntry -> { + EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId()); + + if (entityLogEntry != null) { var request = RemoveRedactionRequest.builder() .annotationId(removeRedactionRequest.getAnnotationId()) .user(KeycloakSecurity.getUserId()) @@ -111,7 +120,7 @@ public class ManualRedactionMapper { } requests.add(RequestEntryPair.builder().request(request.build()).entityLogEntry(entityLogEntry).build()); - }); + } } return requests; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 2fadcf156..215d14ae0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.when; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -56,6 +57,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.Dictionary; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -412,8 +414,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); @@ -471,8 +473,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .removeFromDictionary(true) .removeFromAllDossiers(true) .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); @@ -535,7 +537,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier") .dossierDictionaryEntry(true) @@ -543,7 +546,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -562,7 +566,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -580,7 +585,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier 1 var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .comment("resized dossier redaction") .value("test redaction in dossier dictionary") .updateDictionary(true) @@ -700,7 +706,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier yayy") .dossierDictionaryEntry(true) @@ -708,7 +715,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -727,7 +735,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -745,7 +754,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier 1 var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .comment("resized dossier redaction") .value("test redaction in dossier") .updateDictionary(true) @@ -869,7 +879,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier") .dossierDictionaryEntry(true) @@ -877,7 +888,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -896,7 +908,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template") .dictionaryEntry(true) @@ -914,7 +927,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier dict var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .comment("resized dossier template redaction") .value("test redaction in dossier template dictionary") .updateDictionary(true) @@ -1035,7 +1049,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog1 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(0).getAnnotationId()) .type(typeDosDict.getType()) .value("test redaction in dossier yayy") .dossierDictionaryEntry(true) @@ -1043,7 +1058,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .state(EntryState.APPLIED) .build(), EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -1062,7 +1078,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { var entityLog2 = new EntityLog(1, 1, List.of(EntityLogEntry.builder() - .id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .id(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .type(typeDosTempDict.getType()) .value("test redaction in dossier template yayy") .dictionaryEntry(true) @@ -1080,7 +1097,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { // resize redaction in dossier dict var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() - .annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId()) + .annotationId(addRedactions.getManualAddResponses() + .get(1).getAnnotationId()) .comment("resized dossier template redaction") .value("test redaction in dossier template") .updateDictionary(true) @@ -2039,8 +2057,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .stream() .anyMatch(entry -> entry.getAnnotationId().equals("annotationId"))); - assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false) - .getManualAddResponses().get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long"); + assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses() + .get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long"); assertNull(allManualRedactions.getRecategorizations() .stream() @@ -2504,7 +2522,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .entryType(EntryType.ENTITY) .state(EntryState.APPLIED) .dictionaryEntry(true) - .positions(List.of(new Position(10f, 10f ,10f, 10f, 1))) + .positions(List.of(new Position(10f, 10f, 10f, 10f, 1))) .build()), null, 0, @@ -2515,22 +2533,22 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(), - file.getId(), - Set.of(AddRedactionRequestModel.builder() - .sourceId("") - .reason("Reason") - .type(type.getType()) - .addToAllDossiers(false) - .addToDictionary(false) - .dictionaryEntryType(DictionaryEntryType.ENTRY) - .forceAddToDictionary(false) - .legalBasis("legalBasis") - .positions(List.of(new Rectangle(10f, 10f ,10f, 10f, 1))) - .value("MyValue") - .build())) - .getManualAddResponses().get(0); + file.getId(), + Set.of(AddRedactionRequestModel.builder() + .sourceId("") + .reason("Reason") + .type(type.getType()) + .addToAllDossiers(false) + .addToDictionary(false) + .dictionaryEntryType(DictionaryEntryType.ENTRY) + .forceAddToDictionary(false) + .legalBasis("legalBasis") + .positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1))) + .value("MyValue") + .build())).getManualAddResponses() + .get(0); - assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); + assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); assertEquals(response.getEntityLogEntry().getValue(), "MyValue"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis"); assertEquals(response.getEntityLogEntry().getReason(), "Reason"); @@ -2561,7 +2579,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .entryType(EntryType.ENTITY) .state(EntryState.SKIPPED) .dictionaryEntry(true) - .positions(List.of(new Position(10f, 10f ,10f, 10f, 1))) + .positions(List.of(new Position(10f, 10f, 10f, 10f, 1))) .build()), null, 0, @@ -2572,14 +2590,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(), - file.getId(), - Set.of(ForceRedactionRequestModel.builder() - .annotationId("AnnotationId") - .legalBasis("legalBasis") - .build())) - .getManualAddResponses().get(0); + file.getId(), + Set.of(ForceRedactionRequestModel.builder() + .annotationId("AnnotationId") + .legalBasis("legalBasis") + .build())).getManualAddResponses() + .get(0); - assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); + assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis"); assertEquals(response.getEntityLogEntry().getState(), EntryState.APPLIED); assertEquals(response.getEntityLogEntry().getManualChanges() @@ -2617,14 +2635,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(), - file.getId(), - Set.of(LegalBasisChangeRequestModel.builder() - .annotationId("AnnotationId") - .legalBasis("new legal basis") - .value("Luke Skywalker") - .section("") - .build())) - .getManualAddResponses().get(0); + file.getId(), + Set.of(LegalBasisChangeRequestModel.builder() + .annotationId("AnnotationId") + .legalBasis("new legal basis") + .value("Luke Skywalker") + .section("") + .build())).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis"); @@ -2634,6 +2652,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .get(0).getProcessedDate()); } + @Test public void testLocalResize() { @@ -2652,7 +2671,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .entryType(EntryType.ENTITY) .state(EntryState.APPLIED) .dictionaryEntry(true) - .positions(List.of(new Position(10f, 10f ,10f, 10f, 1))) + .positions(List.of(new Position(10f, 10f, 10f, 10f, 1))) .build()), null, 0, @@ -2663,23 +2682,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); ManualAddResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(), - file.getId(), - Set.of(ResizeRedactionRequestModel.builder() - .annotationId("AnnotationId") - .updateDictionary(false) - .addToAllDossiers(false) - .value("Luke") - .positions(List.of(new Rectangle(5f, 5f, 5f ,5f ,1))) - .build()), - false) - .getManualAddResponses().get(0); + file.getId(), + Set.of(ResizeRedactionRequestModel.builder() + .annotationId("AnnotationId") + .updateDictionary(false) + .addToAllDossiers(false) + .value("Luke") + .positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1))) + .build()), + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getValue(), "Luke"); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[0], 5f); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[1], 5f); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[2], 5f); - assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[3], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[0], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[1], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[2], 5f); + assertEquals(response.getEntityLogEntry().getPositions() + .get(0).getRectangle()[3], 5f); assertEquals(response.getEntityLogEntry().getManualChanges() .get(0).getManualRedactionType(), ManualRedactionType.RESIZE); assertNull(response.getEntityLogEntry().getManualChanges() @@ -2726,8 +2749,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .section("section") .value("Luke Skywalker") .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getType(), type2.getType()); @@ -2772,8 +2795,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .removeFromDictionary(false) .removeFromAllDossiers(false) .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED); @@ -2829,8 +2852,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .section("section") .value("Luke Skywalker") .build()), - false) - .getManualAddResponses().get(0); + false).getManualAddResponses() + .get(0); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getType(), type2.getType()); @@ -2843,4 +2866,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .get(0).getProcessedDate()); } + + @Test + public void testRemoveMoreThan500Annotations() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + List entityLogEntries = new ArrayList<>(); + Set removeRedactionRequestModels = new HashSet<>(); + for (int i = 0; i < 600; i++) { + entityLogEntries.add(EntityLogEntry.builder() + .id("AnnotationId" + i) + .type("test") + .value("Luke Skywalker" + i) + .entryType(EntryType.ENTITY) + .state(EntryState.APPLIED) + .dictionaryEntry(false) + .section("section") + .legalBasis("Article legal basis") + .build()); + removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder() + .annotationId("AnnotationId" + i) + .removeFromDictionary(false) + .removeFromAllDossiers(false) + .build()); + } + var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog); + + ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false); + assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600); + } + } -- 2.47.2