RED-9606 - Fix removal #586

Merged
andrei.isvoran.ext merged 1 commits from RED-9606 into master 2024-07-11 14:36:34 +02:00
2 changed files with 145 additions and 77 deletions

View File

@ -4,9 +4,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
@ -91,10 +94,16 @@ public class ManualRedactionMapper {
.map(RemoveRedactionRequestModel::getAnnotationId)
.toList(),
includeUnprocessed);
Map<String, EntityLogEntry> iddToEntityLogEntryMap = entityLogEntries.stream()
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
List<RequestEntryPair<RemoveRedactionRequest>> requests = new ArrayList<>();
for (var removeRedactionRequest : removeRedactionRequests) {
entityLogEntries.forEach(entityLogEntry -> {
EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId());
if (entityLogEntry != null) {
var request = RemoveRedactionRequest.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
@ -111,7 +120,7 @@ public class ManualRedactionMapper {
}
requests.add(RequestEntryPair.<RemoveRedactionRequest>builder().request(request.build()).entityLogEntry(entityLogEntry).build());
});
}
}
return requests;

View File

@ -16,6 +16,7 @@ import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -56,6 +57,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.Dictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -412,8 +414,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@ -471,8 +473,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(true)
.removeFromAllDossiers(true)
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
assertThat(dossierDictionary.getEntries().size()).isEqualTo(1);
@ -535,7 +537,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier")
.dossierDictionaryEntry(true)
@ -543,7 +546,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -562,7 +566,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -580,7 +585,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction")
.value("test redaction in dossier dictionary")
.updateDictionary(true)
@ -700,7 +706,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier yayy")
.dossierDictionaryEntry(true)
@ -708,7 +715,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -727,7 +735,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -745,7 +754,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction")
.value("test redaction in dossier")
.updateDictionary(true)
@ -869,7 +879,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier")
.dossierDictionaryEntry(true)
@ -877,7 +888,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -896,7 +908,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -914,7 +927,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction")
.value("test redaction in dossier template dictionary")
.updateDictionary(true)
@ -1035,7 +1049,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier yayy")
.dossierDictionaryEntry(true)
@ -1043,7 +1058,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -1062,7 +1078,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -1080,7 +1097,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction")
.value("test redaction in dossier template")
.updateDictionary(true)
@ -2039,8 +2057,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals("annotationId")));
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false)
.getManualAddResponses().get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses()
.get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertNull(allManualRedactions.getRecategorizations()
.stream()
@ -2504,7 +2522,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2515,22 +2533,22 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(),
file.getId(),
Set.of(AddRedactionRequestModel.builder()
.sourceId("")
.reason("Reason")
.type(type.getType())
.addToAllDossiers(false)
.addToDictionary(false)
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.forceAddToDictionary(false)
.legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f ,10f, 10f, 1)))
.value("MyValue")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(AddRedactionRequestModel.builder()
.sourceId("")
.reason("Reason")
.type(type.getType())
.addToAllDossiers(false)
.addToDictionary(false)
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.forceAddToDictionary(false)
.legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1)))
.value("MyValue")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getValue(), "MyValue");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
assertEquals(response.getEntityLogEntry().getReason(), "Reason");
@ -2561,7 +2579,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.SKIPPED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2572,14 +2590,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(),
file.getId(),
Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("legalBasis")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("legalBasis")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
assertEquals(response.getEntityLogEntry().getState(), EntryState.APPLIED);
assertEquals(response.getEntityLogEntry().getManualChanges()
@ -2617,14 +2635,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(),
file.getId(),
Set.of(LegalBasisChangeRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("new legal basis")
.value("Luke Skywalker")
.section("")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(LegalBasisChangeRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("new legal basis")
.value("Luke Skywalker")
.section("")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis");
@ -2634,6 +2652,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate());
}
@Test
public void testLocalResize() {
@ -2652,7 +2671,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2663,23 +2682,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(ResizeRedactionRequestModel.builder()
.annotationId("AnnotationId")
.updateDictionary(false)
.addToAllDossiers(false)
.value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f ,5f ,1)))
.build()),
false)
.getManualAddResponses().get(0);
file.getId(),
Set.of(ResizeRedactionRequestModel.builder()
.annotationId("AnnotationId")
.updateDictionary(false)
.addToAllDossiers(false)
.value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1)))
.build()),
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getValue(), "Luke");
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertNull(response.getEntityLogEntry().getManualChanges()
@ -2726,8 +2749,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section")
.value("Luke Skywalker")
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2772,8 +2795,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED);
@ -2829,8 +2852,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section")
.value("Luke Skywalker")
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2843,4 +2866,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate());
}
@Test
public void testRemoveMoreThan500Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<RemoveRedactionRequestModel> removeRedactionRequestModels = new HashSet<>();
for (int i = 0; i < 600; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(false)
.section("section")
.legalBasis("Article legal basis")
.build());
removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder()
.annotationId("AnnotationId" + i)
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false);
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
}