RED-9606 - Bulk actions on many annotations

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java

@@ -204,11 +204,15 @@ public class ManualRedactionMapper {
                                                                                                               .toList(),
                                                                                                       includeUnprocessed);
 
+        Map<String, EntityLogEntry> annotationIdToEntityLogEntryMap = entityLogEntries.stream()
+                .collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
         List<RequestEntryPair<RecategorizationRequest>> requests = new ArrayList<>();
 
         for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) {
 
-            entityLogEntries.forEach(entityLogEntry -> {
+            EntityLogEntry entityLogEntry = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId());
+
+            if (entityLogEntry != null) {
                 String changedValue;
                 String changedTypeId;
                 String uuid = UUID.randomUUID().toString();
@@ -255,7 +259,7 @@ public class ManualRedactionMapper {
                 }
 
                 requests.add(RequestEntryPair.<RecategorizationRequest>builder().request(request).entityLogEntry(entityLogEntry).build());
-            });
+            }
         }
 
         return requests;
@@ -319,10 +323,7 @@ public class ManualRedactionMapper {
                         .addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
                         .build();
 
-                if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
+                if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary() && entryIsEntityType(entityLogEntry)) {
-                    && !request.isAddToAllDossiers()
-                    && !request.getUpdateDictionary()
-                    && entryIsEntityType(entityLogEntry)) {
                     request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId());
                     entityLogEntry.setId(uuid);
                     manualRedactionEntryConsumer.accept(entityLogEntry);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java

@@ -6,6 +6,7 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC
 import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -197,6 +198,12 @@ public class ManualRedactionService {
                                                       String dossierTemplateId,
                                                       boolean includeUnprocessed) {
 
+        var numberOfDictRemoves = removeRedactionRequests.stream()
+                .filter(removeRedactionRequestModel -> removeRedactionRequestModel.isRemoveFromDictionary() || removeRedactionRequestModel.isRemoveFromAllDossiers())
+                .count();
+        if (numberOfDictRemoves > 100) {
+            throw new BadRequestException("Maximum number of remove from dictionary requests is 100.");
+        }
         var response = new ArrayList<ManualAddResponse>();
         List<RequestEntryPair<RemoveRedactionRequest>> requests = manualRedactionMapper.toRemoveRedactionRequestList(dossierId,
                                                                                                                      fileId,
@@ -204,13 +211,18 @@ public class ManualRedactionService {
                                                                                                                      removeRedactionRequests,
                                                                                                                      includeUnprocessed);
 
+        Set<String> verifiedTypes = new HashSet<>();
         //validate removing from dossier template dictionary
         for (RequestEntryPair<RemoveRedactionRequest> removeRedactionRequestRequestEntryPair : requests) {
             RemoveRedactionRequest removeRedactionRequest = removeRedactionRequestRequestEntryPair.getRequest();
             EntityLogEntry entityLogEntry = removeRedactionRequestRequestEntryPair.getEntityLogEntry();
-            manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest,
+
-                                                                                 removeRedactionRequest.getTypeToRemove(),
+            if (verifiedTypes.add(removeRedactionRequest.getTypeToRemove())) {
-                                                                                 removeRedactionRequest.getDossierTemplateId());
+                manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest,
+                                                                                     removeRedactionRequest.getTypeToRemove(),
+                                                                                     removeRedactionRequest.getDossierTemplateId());
+            }
+
 
             log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
             IdRemovalEntity idRemoval = removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -1847,20 +1847,29 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
 
         ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(),
                                                                                   file.getId(),
-                                                                                  Set.of(RecategorizationRequestModel.builder().annotationId("dv").legalBasis("").section("section").type(type.getType()).build()),
+                                                                                  Set.of(RecategorizationRequestModel.builder()
+                                                                                                 .annotationId("dv")
+                                                                                                 .legalBasis("")
+                                                                                                 .section("section")
+                                                                                                 .type(type.getType())
+                                                                                                 .build()),
                                                                                   false);
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(allManualRedactions.getRecategorizations().size(), 1);
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
 
         var unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
         assertEquals(unprocessedManualRedactions.getRecategorizations().size(), 1);
         assertTrue(unprocessedManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
 
         fileProcessingClient.analysisSuccessful(dossier.getId(),
                                                 file.getId(),
@@ -1873,27 +1882,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                                         .build());
 
         ManualRedactionResponse response2 = manualRedactionClient.recategorizeBulk(dossier.getId(),
-                                               file.getId(),
+                                                                                   file.getId(),
-                                               Set.of(RecategorizationRequestModel.builder().annotationId("dv2").legalBasis("").section("section").type(type.getType()).build()),
+                                                                                   Set.of(RecategorizationRequestModel.builder()
-                                               false);
+                                                                                                  .annotationId("dv2")
+                                                                                                  .legalBasis("")
+                                                                                                  .section("section")
+                                                                                                  .type(type.getType())
+                                                                                                  .build()),
+                                                                                   false);
 
         allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(allManualRedactions.getRecategorizations().size(), 2);
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response2.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
 
         unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
         assertEquals(unprocessedManualRedactions.getRecategorizations().size(), 1);
         assertTrue(unprocessedManualRedactions.getRecategorizations()
                            .stream()
-                           .noneMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
+                           .noneMatch(entry -> entry.getAnnotationId()
+                                   .equals(response.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
         assertTrue(unprocessedManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response2.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
 
         fileProcessingClient.analysisSuccessful(dossier.getId(),
                                                 file.getId(),
@@ -1909,10 +1931,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         assertEquals(allManualRedactions.getRecategorizations().size(), 2);
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response2.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
 
         unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
         assertTrue(unprocessedManualRedactions.getRecategorizations().isEmpty());
@@ -2249,13 +2275,15 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                 .section("overriddenSection")
                 .build();
 
-        ManualRedactionResponse response =manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false);
+        ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false);
 
         var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
         assertEquals(1, allManualRedactions.getRecategorizations().size());
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
-                           .anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
+                           .anyMatch(entry -> entry.getAnnotationId()
+                                   .equals(response.getManualAddResponses()
+                                                   .get(0).getAnnotationId())));
         assertTrue(allManualRedactions.getRecategorizations()
                            .stream()
                            .anyMatch(entry -> entry.getLegalBasis().equals("lb2")));
@@ -3028,4 +3056,110 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
         assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
     }
 
+
+    @Test
+    public void testForceMoreThan500Annotations() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+
+        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
+        Set<ForceRedactionRequestModel> forceRedactionRequestModels = new HashSet<>();
+        for (int i = 0; i < 600; i++) {
+            entityLogEntries.add(EntityLogEntry.builder()
+                                         .id("AnnotationId" + i)
+                                         .type("test")
+                                         .value("Luke Skywalker" + i)
+                                         .entryType(EntryType.ENTITY)
+                                         .state(EntryState.SKIPPED)
+                                         .dictionaryEntry(false)
+                                         .section("section")
+                                         .legalBasis("Article legal basis")
+                                         .build());
+            forceRedactionRequestModels.add(ForceRedactionRequestModel.builder().annotationId("AnnotationId" + i).legalBasis("lb1").comment("").build());
+        }
+        var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
+        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
+
+        when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
+
+        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), forceRedactionRequestModels);
+        assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
+    }
+
+
+    @Test
+    public void testRecategorizeMoreThan500Annotations() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+
+        var type = typeProvider.testAndProvideType(dossierTemplate, null, "type", true);
+        assertThat(type.isDossierDictionaryOnly()).isTrue();
+
+        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
+        Set<RecategorizationRequestModel> recategorizationRequestModels = new HashSet<>();
+        for (int i = 0; i < 600; i++) {
+            entityLogEntries.add(EntityLogEntry.builder()
+                                         .id("AnnotationId" + i)
+                                         .type("test")
+                                         .value("Luke Skywalker" + i)
+                                         .entryType(EntryType.ENTITY)
+                                         .state(EntryState.SKIPPED)
+                                         .dictionaryEntry(false)
+                                         .section("section")
+                                         .legalBasis("Article legal basis")
+                                         .build());
+            recategorizationRequestModels.add(RecategorizationRequestModel.builder()
+                                                      .annotationId("AnnotationId" + i)
+                                                      .legalBasis("lb1")
+                                                      .type("type")
+                                                      .addToAllDossiers(false)
+                                                      .addToDictionary(false)
+                                                      .comment("")
+                                                      .build());
+        }
+        var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
+        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
+
+        when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
+
+        ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequestModels, false);
+        assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
+    }
+
+
+    @Test
+    public void testRemoveFromDictMoreThan100Annotations() {
+
+        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
+        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
+        var file = fileTesterAndProvider.testAndProvideFile(dossier);
+
+        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
+        Set<RemoveRedactionRequestModel> removeRedactionRequestModels = new HashSet<>();
+        for (int i = 0; i < 101; i++) {
+            entityLogEntries.add(EntityLogEntry.builder()
+                                         .id("AnnotationId" + i)
+                                         .type("test")
+                                         .value("Luke Skywalker" + i)
+                                         .entryType(EntryType.ENTITY)
+                                         .state(EntryState.APPLIED)
+                                         .dictionaryEntry(true)
+                                         .section("section")
+                                         .legalBasis("Article legal basis")
+                                         .build());
+            removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder().annotationId("AnnotationId" + i).removeFromDictionary(true).removeFromAllDossiers(true).build());
+        }
+        var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
+        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
+
+        when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
+
+        var result = assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false));
+        assertTrue(result.getMessage().contains("Maximum number of remove from dictionary requests is 100."));
+    }
+
 }