Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9606 - Bulk actions on many annotations #600

Merged
andrei.isvoran.ext merged 1 commits from RED-9606-fp into master 2024-07-16 13:25:18 +02:00
Conversation 5 Commits 1 Files Changed 3 +170 -23
3 changed files with 170 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java
View File

@ -204,11 +204,15 @@ public class ManualRedactionMapper {
.toList(),
includeUnprocessed);
Map<String, EntityLogEntry> annotationIdToEntityLogEntryMap = entityLogEntries.stream()
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
List<RequestEntryPair<RecategorizationRequest>> requests = new ArrayList<>();
for (RecategorizationRequestModel recategorizationRequest : recategorizationRequests) {
entityLogEntries.forEach(entityLogEntry -> {
EntityLogEntry entityLogEntry = annotationIdToEntityLogEntryMap.get(recategorizationRequest.getAnnotationId());
if (entityLogEntry != null) {
String changedValue;
String changedTypeId;
String uuid = UUID.randomUUID().toString();
@ -255,7 +259,7 @@ public class ManualRedactionMapper {
}
requests.add(RequestEntryPair.<RecategorizationRequest>builder().request(request).entityLogEntry(entityLogEntry).build());
});
}
}
return requests;
@ -319,10 +323,7 @@ public class ManualRedactionMapper {
.addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
&& !request.isAddToAllDossiers()
&& !request.getUpdateDictionary()
&& entryIsEntityType(entityLogEntry)) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary() && entryIsEntityType(entityLogEntry)) {
request.setBasedOnDictAnnotationId(resizeRedactionRequest.getAnnotationId());
entityLogEntry.setId(uuid);
manualRedactionEntryConsumer.accept(entityLogEntry);

18
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -6,6 +6,7 @@ import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicC
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -197,6 +198,12 @@ public class ManualRedactionService {
String dossierTemplateId,
boolean includeUnprocessed) {
var numberOfDictRemoves = removeRedactionRequests.stream()
.filter(removeRedactionRequestModel -> removeRedactionRequestModel.isRemoveFromDictionary() || removeRedactionRequestModel.isRemoveFromAllDossiers())
.count();
if (numberOfDictRemoves > 100) {
throw new BadRequestException("Maximum number of remove from dictionary requests is 100.");
}
var response = new ArrayList<ManualAddResponse>();
List<RequestEntryPair<RemoveRedactionRequest>> requests = manualRedactionMapper.toRemoveRedactionRequestList(dossierId,
fileId,
@ -204,13 +211,18 @@ public class ManualRedactionService {
removeRedactionRequests,
includeUnprocessed);
Set<String> verifiedTypes = new HashSet<>();
//validate removing from dossier template dictionary
for (RequestEntryPair<RemoveRedactionRequest> removeRedactionRequestRequestEntryPair : requests) {
RemoveRedactionRequest removeRedactionRequest = removeRedactionRequestRequestEntryPair.getRequest();
EntityLogEntry entityLogEntry = removeRedactionRequestRequestEntryPair.getEntityLogEntry();
manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest,
removeRedactionRequest.getTypeToRemove(),
removeRedactionRequest.getDossierTemplateId());
if (verifiedTypes.add(removeRedactionRequest.getTypeToRemove())) {
manualRedactionDictionaryUpdateHandler.validateDictionariesForDelete(removeRedactionRequest,
removeRedactionRequest.getTypeToRemove(),
removeRedactionRequest.getDossierTemplateId());
}
log.info("add removeRedaction for file {} and annotation {}", fileId, removeRedactionRequest.getAnnotationId());
IdRemovalEntity idRemoval = removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);

162
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -1847,20 +1847,29 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(),
file.getId(),
Set.of(RecategorizationRequestModel.builder().annotationId("dv").legalBasis("").section("section").type(type.getType()).build()),
Set.of(RecategorizationRequestModel.builder()
.annotationId("dv")
.legalBasis("")
.section("section")
.type(type.getType())
.build()),
false);
var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
assertEquals(allManualRedactions.getRecategorizations().size(), 1);
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response.getManualAddResponses()
.get(0).getAnnotationId())));
var unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
assertEquals(unprocessedManualRedactions.getRecategorizations().size(), 1);
assertTrue(unprocessedManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response.getManualAddResponses()
.get(0).getAnnotationId())));
fileProcessingClient.analysisSuccessful(dossier.getId(),
file.getId(),
@ -1873,27 +1882,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.build());
ManualRedactionResponse response2 = manualRedactionClient.recategorizeBulk(dossier.getId(),
file.getId(),
Set.of(RecategorizationRequestModel.builder().annotationId("dv2").legalBasis("").section("section").type(type.getType()).build()),
false);
file.getId(),
Set.of(RecategorizationRequestModel.builder()
.annotationId("dv2")
.legalBasis("")
.section("section")
.type(type.getType())
.build()),
false);
allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
assertEquals(allManualRedactions.getRecategorizations().size(), 2);
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response.getManualAddResponses()
.get(0).getAnnotationId())));
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response2.getManualAddResponses()
.get(0).getAnnotationId())));
unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
assertEquals(unprocessedManualRedactions.getRecategorizations().size(), 1);
assertTrue(unprocessedManualRedactions.getRecategorizations()
.stream()
.noneMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
.noneMatch(entry -> entry.getAnnotationId()
.equals(response.getManualAddResponses()
.get(0).getAnnotationId())));
assertTrue(unprocessedManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response2.getManualAddResponses()
.get(0).getAnnotationId())));
fileProcessingClient.analysisSuccessful(dossier.getId(),
file.getId(),
@ -1909,10 +1931,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(allManualRedactions.getRecategorizations().size(), 2);
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response.getManualAddResponses()
.get(0).getAnnotationId())));
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response2.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response2.getManualAddResponses()
.get(0).getAnnotationId())));
unprocessedManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), true, true);
assertTrue(unprocessedManualRedactions.getRecategorizations().isEmpty());
@ -2249,13 +2275,15 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("overriddenSection")
.build();
ManualRedactionResponse response =manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false);
ManualRedactionResponse response = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModel), false);
var allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
assertEquals(1, allManualRedactions.getRecategorizations().size());
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals(response.getManualAddResponses().get(0).getAnnotationId())));
.anyMatch(entry -> entry.getAnnotationId()
.equals(response.getManualAddResponses()
.get(0).getAnnotationId())));
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(entry -> entry.getLegalBasis().equals("lb2")));
@ -3028,4 +3056,110 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
@Test
public void testForceMoreThan500Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<ForceRedactionRequestModel> forceRedactionRequestModels = new HashSet<>();
for (int i = 0; i < 600; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.SKIPPED)
.dictionaryEntry(false)
.section("section")
.legalBasis("Article legal basis")
.build());
forceRedactionRequestModels.add(ForceRedactionRequestModel.builder().annotationId("AnnotationId" + i).legalBasis("lb1").comment("").build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.forceRedactionBulk(dossier.getId(), file.getId(), forceRedactionRequestModels);
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
@Test
public void testRecategorizeMoreThan500Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var type = typeProvider.testAndProvideType(dossierTemplate, null, "type", true);
assertThat(type.isDossierDictionaryOnly()).isTrue();
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<RecategorizationRequestModel> recategorizationRequestModels = new HashSet<>();
for (int i = 0; i < 600; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.SKIPPED)
.dictionaryEntry(false)
.section("section")
.legalBasis("Article legal basis")
.build());
recategorizationRequestModels.add(RecategorizationRequestModel.builder()
.annotationId("AnnotationId" + i)
.legalBasis("lb1")
.type("type")
.addToAllDossiers(false)
.addToDictionary(false)
.comment("")
.build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequestModels, false);
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
@Test
public void testRemoveFromDictMoreThan100Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<RemoveRedactionRequestModel> removeRedactionRequestModels = new HashSet<>();
for (int i = 0; i < 101; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.section("section")
.legalBasis("Article legal basis")
.build());
removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder().annotationId("AnnotationId" + i).removeFromDictionary(true).removeFromAllDossiers(true).build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
var result = assertThrows(FeignException.class, () -> manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false));
assertTrue(result.getMessage().contains("Maximum number of remove from dictionary requests is 100."));
}
}