RED-9606 - Fix removal #585

Merged
andrei.isvoran.ext merged 1 commits from RED-9606-bp into release/2.465.x 2024-07-11 14:36:37 +02:00
2 changed files with 145 additions and 77 deletions

View File

@ -4,9 +4,12 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ty
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
@ -91,10 +94,16 @@ public class ManualRedactionMapper {
.map(RemoveRedactionRequestModel::getAnnotationId)
.toList(),
includeUnprocessed);
Map<String, EntityLogEntry> iddToEntityLogEntryMap = entityLogEntries.stream()
.collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
List<RequestEntryPair<RemoveRedactionRequest>> requests = new ArrayList<>();
for (var removeRedactionRequest : removeRedactionRequests) {
entityLogEntries.forEach(entityLogEntry -> {
EntityLogEntry entityLogEntry = iddToEntityLogEntryMap.get(removeRedactionRequest.getAnnotationId());
if (entityLogEntry != null) {
var request = RemoveRedactionRequest.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
@ -111,7 +120,7 @@ public class ManualRedactionMapper {
}
requests.add(RequestEntryPair.<RemoveRedactionRequest>builder().request(request.build()).entityLogEntry(entityLogEntry).build());
});
}
}
return requests;

View File

@ -15,6 +15,7 @@ import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -55,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.Dictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -411,8 +413,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@ -470,8 +472,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(true)
.removeFromAllDossiers(true)
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
assertThat(dossierDictionary.getEntries().size()).isEqualTo(1);
@ -534,7 +536,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier")
.dossierDictionaryEntry(true)
@ -542,7 +545,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -561,7 +565,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -579,7 +584,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction")
.value("test redaction in dossier dictionary")
.updateDictionary(true)
@ -699,7 +705,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier yayy")
.dossierDictionaryEntry(true)
@ -707,7 +714,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -726,7 +734,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -744,7 +753,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.comment("resized dossier redaction")
.value("test redaction in dossier")
.updateDictionary(true)
@ -868,7 +878,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier")
.dossierDictionaryEntry(true)
@ -876,7 +887,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -895,7 +907,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template")
.dictionaryEntry(true)
@ -913,7 +926,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction")
.value("test redaction in dossier template dictionary")
.updateDictionary(true)
@ -1034,7 +1048,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog1 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(0).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(0).getAnnotationId())
.type(typeDosDict.getType())
.value("test redaction in dossier yayy")
.dossierDictionaryEntry(true)
@ -1042,7 +1057,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.state(EntryState.APPLIED)
.build(),
EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -1061,7 +1077,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var entityLog2 = new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.id(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.type(typeDosTempDict.getType())
.value("test redaction in dossier template yayy")
.dictionaryEntry(true)
@ -1079,7 +1096,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
.annotationId(addRedactions.getManualAddResponses().get(1).getAnnotationId())
.annotationId(addRedactions.getManualAddResponses()
.get(1).getAnnotationId())
.comment("resized dossier template redaction")
.value("test redaction in dossier template")
.updateDictionary(true)
@ -2038,8 +2056,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.stream()
.anyMatch(entry -> entry.getAnnotationId().equals("annotationId")));
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false)
.getManualAddResponses().get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertThatThrownBy(() -> manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), Set.of(recatModelLongLegalBasis), false).getManualAddResponses()
.get(0)).isInstanceOf(FeignException.class).hasMessageContaining("The legal basis is too long");
assertNull(allManualRedactions.getRecategorizations()
.stream()
@ -2503,7 +2521,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2514,22 +2532,22 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.addRedactionBulk(dossier.getId(),
file.getId(),
Set.of(AddRedactionRequestModel.builder()
.sourceId("")
.reason("Reason")
.type(type.getType())
.addToAllDossiers(false)
.addToDictionary(false)
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.forceAddToDictionary(false)
.legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f ,10f, 10f, 1)))
.value("MyValue")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(AddRedactionRequestModel.builder()
.sourceId("")
.reason("Reason")
.type(type.getType())
.addToAllDossiers(false)
.addToDictionary(false)
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.forceAddToDictionary(false)
.legalBasis("legalBasis")
.positions(List.of(new Rectangle(10f, 10f, 10f, 10f, 1)))
.value("MyValue")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getId(), response.getAnnotationId());
assertEquals(response.getEntityLogEntry().getValue(), "MyValue");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
assertEquals(response.getEntityLogEntry().getReason(), "Reason");
@ -2560,7 +2578,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.SKIPPED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2571,14 +2589,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.forceRedactionBulk(dossier.getId(),
file.getId(),
Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("legalBasis")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(ForceRedactionRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("legalBasis")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "legalBasis");
assertEquals(response.getEntityLogEntry().getState(), EntryState.APPLIED);
assertEquals(response.getEntityLogEntry().getManualChanges()
@ -2616,14 +2634,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.legalBasisChangeBulk(dossier.getId(),
file.getId(),
Set.of(LegalBasisChangeRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("new legal basis")
.value("Luke Skywalker")
.section("")
.build()))
.getManualAddResponses().get(0);
file.getId(),
Set.of(LegalBasisChangeRequestModel.builder()
.annotationId("AnnotationId")
.legalBasis("new legal basis")
.value("Luke Skywalker")
.section("")
.build())).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getLegalBasis(), "new legal basis");
@ -2633,6 +2651,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate());
}
@Test
public void testLocalResize() {
@ -2651,7 +2670,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(List.of(new Position(10f, 10f ,10f, 10f, 1)))
.positions(List.of(new Position(10f, 10f, 10f, 10f, 1)))
.build()),
null,
0,
@ -2662,23 +2681,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualAddResponse response = manualRedactionClient.resizeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(ResizeRedactionRequestModel.builder()
.annotationId("AnnotationId")
.updateDictionary(false)
.addToAllDossiers(false)
.value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f ,5f ,1)))
.build()),
false)
.getManualAddResponses().get(0);
file.getId(),
Set.of(ResizeRedactionRequestModel.builder()
.annotationId("AnnotationId")
.updateDictionary(false)
.addToAllDossiers(false)
.value("Luke")
.positions(List.of(new Rectangle(5f, 5f, 5f, 5f, 1)))
.build()),
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getValue(), "Luke");
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions().get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[0], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[1], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[2], 5f);
assertEquals(response.getEntityLogEntry().getPositions()
.get(0).getRectangle()[3], 5f);
assertEquals(response.getEntityLogEntry().getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertNull(response.getEntityLogEntry().getManualChanges()
@ -2725,8 +2748,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section")
.value("Luke Skywalker")
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2771,8 +2794,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getState(), EntryState.IGNORED);
@ -2828,8 +2851,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section("section")
.value("Luke Skywalker")
.build()),
false)
.getManualAddResponses().get(0);
false).getManualAddResponses()
.get(0);
assertEquals(response.getEntityLogEntry().getId(), "AnnotationId");
assertEquals(response.getEntityLogEntry().getType(), type2.getType());
@ -2842,4 +2865,40 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.get(0).getProcessedDate());
}
@Test
public void testRemoveMoreThan500Annotations() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
Set<RemoveRedactionRequestModel> removeRedactionRequestModels = new HashSet<>();
for (int i = 0; i < 600; i++) {
entityLogEntries.add(EntityLogEntry.builder()
.id("AnnotationId" + i)
.type("test")
.value("Luke Skywalker" + i)
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(false)
.section("section")
.legalBasis("Article legal basis")
.build());
removeRedactionRequestModels.add(RemoveRedactionRequestModel.builder()
.annotationId("AnnotationId" + i)
.removeFromDictionary(false)
.removeFromAllDossiers(false)
.build());
}
var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.removeRedactionBulk(dossier.getId(), file.getId(), removeRedactionRequestModels, false);
assertEquals(manualRedactionResponse.getManualAddResponses().size(), 600);
}
}