Merge branch 'RED-9606' into 'master'

RED-9606 - Fix removal

Closes RED-9606

See merge request redactmanager/persistence-service!586
This commit is contained in:
Andrei Isvoran 2024-07-11 14:36:33 +02:00
commit 6f5174cc4b
2 changed files with 144 additions and 77 deletions

View File

@ -4,10 +4,13 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
@ -92,10 +95,16 @@ public class ManualRedactionMapper {
.map(RemoveRedactionRequestModel::getAnnotationId)
.toList(),
includeUnprocessed);
Map<String, EntityLogEntry> iddToEntityLogEntryMap = entityLogEntries.stream()
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
List<RequestEntryPair<RemoveRedactionRequest>> requests = new ArrayList<>();
for (var removeRedactionRequest : removeRedactionRequests) {
entityLogEntries.forEach(entityLogEntry -> {
EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId());
if (entityLogEntry != null) {
var request = RemoveRedactionRequest.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
@ -112,7 +121,7 @@ public class ManualRedactionMapper {
}
requests.add(RequestEntryPair.<RemoveRedactionRequest>builder().request(request.build()).entityLogEntry(entityLogEntry).build());
});
}
}
return requests;

View File

@ -17,6 +17,7 @@ import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -414,8 +415,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@ -473,8 +474,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(true)
.removeFromAllDossiers(true)
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
assertThat(dossierDictionary.getEntries().size()).isEqualTo(1);
@ -537,7 +538,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier")
.dossierDictionaryEntry(true)
@ -545,7 +547,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -564,7 +567,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -582,7 +586,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction")
.value("test redaction in dossier dictionary")
.updateDictionary(true)
@ -702,7 +707,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier yayy")
.dossierDictionaryEntry(true)
@ -710,7 +716,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -729,7 +736,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -747,7 +755,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction")
.value("test redaction in dossier")
.updateDictionary(true)
@ -871,7 +880,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier")
.dossierDictionaryEntry(true)
@ -879,7 +889,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -898,7 +909,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -916,7 +928,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction")
.value("test redaction in dossier template dictionary")
.updateDictionary(true)
@ -1037,7 +1050,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier yayy")
.dossierDictionaryEntry(true)
@ -1045,7 +1059,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -1064,7 +1079,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -1082,7 +1098,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction")
.value("test redaction in dossier template")
.updateDictionary(true)
@ -2041,8 +2058,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals("annotationId")));
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false)
.getManualAddResponses().get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses()
.get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertNull(allManualRedactions.getRecategorizations()
.stream()
@ -2506,7 +2523,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2517,22 +2534,22 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(),
file.getId(),
Set.of(AddRedactionRequestModel.builder()
.sourceId("")
.reason("Reason")
.type(type.getType())
.addToAllDossiers(false)
.addToDictionary(false)
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.forceAddToDictionary(false)
.legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f ,10f, 10f, 1)))
.value("MyValue")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(AddRedactionRequestModel.builder()
.sourceId("")
.reason("Reason")
.type(type.getType())
.addToAllDossiers(false)
.addToDictionary(false)
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.forceAddToDictionary(false)
.legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1)))
.value("MyValue")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getValue(), "MyValue");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
assertEquals(response.getEntityLogEntry().getReason(), "Reason");
@ -2563,7 +2580,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.SKIPPED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2574,14 +2591,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(),
file.getId(),
Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("legalBasis")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("legalBasis")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
assertEquals(response.getEntityLogEntry().getState(), EntryState.APPLIED);
assertEquals(response.getEntityLogEntry().getManualChanges()
@ -2619,14 +2636,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(),
file.getId(),
Set.of(LegalBasisChangeRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("new legal basis")
.value("Luke Skywalker")
.section("")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(LegalBasisChangeRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("new legal basis")
.value("Luke Skywalker")
.section("")
.build())).getManualAddResponses()
.get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis");
@ -2636,6 +2653,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate());
}
@Test
public void testLocalResizeOnDictEntry() {
@ -2654,7 +2672,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2665,23 +2683,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(ResizeRedactionRequestModel.builder()
.annotationId("AnnotationId")
.updateDictionary(false)
.addToAllDossiers(false)
.value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f ,5f ,1)))
.build()),
false)
.getManualAddResponses().get(0);
file.getId(),
Set.of(ResizeRedactionRequestModel.builder()
.annotationId("AnnotationId")
.updateDictionary(false)
.addToAllDossiers(false)
.value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1)))
.build()),
false).getManualAddResponses()
.get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getValue(), "Luke");
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertNull(response.getEntityLogEntry().getManualChanges()
@ -2728,8 +2750,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section")
.value("Luke Skywalker")
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2774,8 +2796,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED);
@ -2831,8 +2853,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section")
.value("Luke Skywalker")
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2845,4 +2867,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate());
}
@Test
public void testRemoveMoreThan500Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<RemoveRedactionRequestModel> removeRedactionRequestModels = new HashSet<>();
for (int i = 0; i < 600; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(false)
.section("section")
.legalBasis("Article legal basis")
.build());
removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder()
.annotationId("AnnotationId" + i)
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false);
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
}