Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-9606' into 'master'

Browse Source 
RED-9606 - Fix removal

Closes RED-9606

See merge request redactmanager/persistence-service!586
This commit is contained in:
Andrei Isvoran 2024-07-11 14:36:33 +02:00
commit 6f5174cc4b
2 changed files with 144 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java
View File

@ -4,10 +4,13 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -92,10 +95,16 @@ public class ManualRedactionMapper {
.map(RemoveRedactionRequestModel::getAnnotationId) .map(RemoveRedactionRequestModel::getAnnotationId)
.toList(), .toList(),
includeUnprocessed); includeUnprocessed);
Map<String, EntityLogEntry> iddToEntityLogEntryMap = entityLogEntries.stream()
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
List<RequestEntryPair<RemoveRedactionRequest>> requests = new ArrayList<>(); List<RequestEntryPair<RemoveRedactionRequest>> requests = new ArrayList<>();
for (var removeRedactionRequest : removeRedactionRequests) { for (var removeRedactionRequest : removeRedactionRequests) {
entityLogEntries.forEach(entityLogEntry -> { EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId());
if (entityLogEntry != null) {
var request = RemoveRedactionRequest.builder() var request = RemoveRedactionRequest.builder()
.annotationId(removeRedactionRequest.getAnnotationId()) .annotationId(removeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId()) .user(KeycloakSecurity.getUserId())
@ -112,7 +121,7 @@ public class ManualRedactionMapper {
} }
requests.add(RequestEntryPair.<RemoveRedactionRequest>builder().request(request.build()).entityLogEntry(entityLogEntry).build()); requests.add(RequestEntryPair.<RemoveRedactionRequest>builder().request(request.build()).entityLogEntry(entityLogEntry).build());
}); }
} }
return requests; return requests;

138
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -17,6 +17,7 @@ import static org.mockito.Mockito.when;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -414,8 +415,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.removeRedactionBulk(dossier.getId(), manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(), file.getId(),
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()), Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()),
false) false).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@ -473,8 +474,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(true) .removeFromDictionary(true)
.removeFromAllDossiers(true) .removeFromAllDossiers(true)
.build()), .build()),
false) false).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null); var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
assertThat(dossierDictionary.getEntries().size()).isEqualTo(1); assertThat(dossierDictionary.getEntries().size()).isEqualTo(1);
@ -537,7 +538,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1, var entityLog1 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType()) .type(typeDosDict.getType())
.value("test redaction in dossier") .value("test redaction in dossier")
.dossierDictionaryEntry(true) .dossierDictionaryEntry(true)
@ -545,7 +547,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED) .state(EntryState.APPLIED)
.build(), .build(),
EntityLogEntry.builder() EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template") .value("test redaction in dossier template")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -564,7 +567,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1, var entityLog2 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template") .value("test redaction in dossier template")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -582,7 +586,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1 // resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId()) .annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction") .comment("resized dossier redaction")
.value("test redaction in dossier dictionary") .value("test redaction in dossier dictionary")
.updateDictionary(true) .updateDictionary(true)
@ -702,7 +707,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1, var entityLog1 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType()) .type(typeDosDict.getType())
.value("test redaction in dossier yayy") .value("test redaction in dossier yayy")
.dossierDictionaryEntry(true) .dossierDictionaryEntry(true)
@ -710,7 +716,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED) .state(EntryState.APPLIED)
.build(), .build(),
EntityLogEntry.builder() EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy") .value("test redaction in dossier template yayy")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -729,7 +736,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1, var entityLog2 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy") .value("test redaction in dossier template yayy")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -747,7 +755,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1 // resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder() var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId()) .annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction") .comment("resized dossier redaction")
.value("test redaction in dossier") .value("test redaction in dossier")
.updateDictionary(true) .updateDictionary(true)
@ -871,7 +880,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1, var entityLog1 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType()) .type(typeDosDict.getType())
.value("test redaction in dossier") .value("test redaction in dossier")
.dossierDictionaryEntry(true) .dossierDictionaryEntry(true)
@ -879,7 +889,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED) .state(EntryState.APPLIED)
.build(), .build(),
EntityLogEntry.builder() EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template") .value("test redaction in dossier template")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -898,7 +909,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1, var entityLog2 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template") .value("test redaction in dossier template")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -916,7 +928,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict // resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction") .comment("resized dossier template redaction")
.value("test redaction in dossier template dictionary") .value("test redaction in dossier template dictionary")
.updateDictionary(true) .updateDictionary(true)
@ -1037,7 +1050,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1, var entityLog1 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType()) .type(typeDosDict.getType())
.value("test redaction in dossier yayy") .value("test redaction in dossier yayy")
.dossierDictionaryEntry(true) .dossierDictionaryEntry(true)
@ -1045,7 +1059,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED) .state(EntryState.APPLIED)
.build(), .build(),
EntityLogEntry.builder() EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy") .value("test redaction in dossier template yayy")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -1064,7 +1079,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1, var entityLog2 = new EntityLog(1,
1, 1,
List.of(EntityLogEntry.builder() List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType()) .type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy") .value("test redaction in dossier template yayy")
.dictionaryEntry(true) .dictionaryEntry(true)
@ -1082,7 +1098,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict // resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder() var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId()) .annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction") .comment("resized dossier template redaction")
.value("test redaction in dossier template") .value("test redaction in dossier template")
.updateDictionary(true) .updateDictionary(true)
@ -2041,8 +2058,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.stream() .stream()
.anyMatch(entry -> entry.getAnnotationId().equals("annotationId"))); .anyMatch(entry -> entry.getAnnotationId().equals("annotationId")));
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false) assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses()
.getManualAddResponses().get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long"); .get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertNull(allManualRedactions.getRecategorizations() assertNull(allManualRedactions.getRecategorizations()
.stream() .stream()
@ -2529,8 +2546,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.legalBasis("legalBasis") .legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1))) .positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1)))
.value("MyValue") .value("MyValue")
.build())) .build())).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId()); assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getValue(), "MyValue"); assertEquals(response.getEntityLogEntry().getValue(), "MyValue");
@ -2578,8 +2595,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
Set.of(ForceRedactionRequestModel.builder() Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId") .annotationId("AnnotationId")
.legalBasis("legalBasis") .legalBasis("legalBasis")
.build())) .build())).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
@ -2625,8 +2642,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.legalBasis("new legal basis") .legalBasis("new legal basis")
.value("Luke Skywalker") .value("Luke Skywalker")
.section("") .section("")
.build())) .build())).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis"); assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis");
@ -2636,6 +2653,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate()); .get(0).getProcessedDate());
} }
@Test @Test
public void testLocalResizeOnDictEntry() { public void testLocalResizeOnDictEntry() {
@ -2673,15 +2691,19 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.value("Luke") .value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1))) .positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1)))
.build()), .build()),
false) false).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getValue(), "Luke"); assertEquals(response.getEntityLogEntry().getValue(), "Luke");
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[0], 5f); assertEquals(response.getEntityLogEntry().getPositions()
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[1], 5f); .get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[2], 5f); assertEquals(response.getEntityLogEntry().getPositions()
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[3], 5f); .get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getManualChanges() assertEquals(response.getEntityLogEntry().getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RESIZE); .get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertNull(response.getEntityLogEntry().getManualChanges() assertNull(response.getEntityLogEntry().getManualChanges()
@ -2728,8 +2750,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section") .section("section")
.value("Luke Skywalker") .value("Luke Skywalker")
.build()), .build()),
false) false).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType()); assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2774,8 +2796,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(false) .removeFromDictionary(false)
.removeFromAllDossiers(false) .removeFromAllDossiers(false)
.build()), .build()),
false) false).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED); assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED);
@ -2831,8 +2853,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section") .section("section")
.value("Luke Skywalker") .value("Luke Skywalker")
.build()), .build()),
false) false).getManualAddResponses()
.getManualAddResponses().get(0); .get(0);
assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId"); assertNotEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType()); assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2845,4 +2867,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate()); .get(0).getProcessedDate());
} }
@Test
public void testRemoveMoreThan500Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<RemoveRedactionRequestModel> removeRedactionRequestModels = new HashSet<>();
for (int i = 0; i < 600; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(false)
.section("section")
.legalBasis("Article legal basis")
.build());
removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder()
.annotationId("AnnotationId" + i)
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false);
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
} }