RED-7782 - Merge unprocessed redactions into entity log without re-analysis.

This commit is contained in:
Andrei Isvoran 2023-10-26 15:37:33 +03:00
parent 087935dddd
commit 75847a43d5
11 changed files with 589 additions and 61 deletions

View File

@ -27,9 +27,10 @@ public class EntityLogController implements EntityLogResource {
@PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
public EntityLog getEntityLog(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes) {
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
@RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean unprocessed) {
return entityLogService.getEntityLog(dossierId, fileId, excludedTypes);
return entityLogService.getEntityLog(dossierId, fileId, excludedTypes, unprocessed);
}

View File

@ -29,13 +29,16 @@ public interface EntityLogResource {
String DOSSIER_ID = "dossierId";
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
String FALSE = "false";
@GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Gets the entity log for a fileId", description = "None")
@Operation(summary = "Gets the entity log for a fileId", description = "Gets the entity log for a given file. The flag includeUnprocessed will merge into the entity log all the unprocessed changes if it's set to true." +
"Default value for the flag is false.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The entity log is not found.")})
EntityLog getEntityLog(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes);
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
@RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
@PostMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + "/filtered", produces = MediaType.APPLICATION_JSON_VALUE)

View File

@ -21,9 +21,10 @@ public class EntityLogInternalController implements EntityLogResource {
public EntityLog getEntityLog(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes) {
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
@RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
return entityLogService.getEntityLog(dossierId, fileId, excludedTypes);
return entityLogService.getEntityLog(dossierId, fileId, excludedTypes, includeUnprocessed);
}

View File

@ -26,11 +26,14 @@ public interface EntityLogResource {
String DOSSIER_ID = "dossierId";
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
String FALSE = "false";
@GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Gets the entity log for a fileId", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The entity log is not found.")})
EntityLog getEntityLog(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes);
@RequestParam(value = "excludedType", required = false) List<String> excludedTypes,
@RequestParam(value = "unprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
}

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.client.redactionservice;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
@FeignClient(name = "UnprocessedManualEntityClient", url = "${redaction-service.url}")
public interface UnprocessedManualEntityClient extends UnprocessedManualEntityResource {
}

View File

@ -3,37 +3,63 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Slf4j
public class EntityLogService {
FileManagementStorageService fileManagementStorageService;
FileStatusService fileStatusService;
ManualRedactionProviderService manualRedactionProviderService;
UnprocessedManualEntityClient unprocessedManualEntityClient;
private final DossierService dossierService;
CommentService commentService;
public EntityLog getEntityLog(String dossierId, String fileId) {
return getEntityLog(dossierId, fileId, Collections.emptyList());
return getEntityLog(dossierId, fileId, Collections.emptyList(), false);
}
public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes) {
public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes, boolean includeUnprocessed) {
var fileStatus = fileStatusService.getStatus(fileId);
@ -49,6 +75,12 @@ public class EntityLogService {
entityLog.getEntityLogEntry().removeIf(entry -> excludedTypes.contains(entry.getType()));
}
if (includeUnprocessed) {
ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
List<UnprocessedManualEntity> unprocessedManualEntities = getUnprocessedManualEntities(dossierId, fileId, manualRedactions);
mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog);
}
Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);
entityLog.getEntityLogEntry().forEach(entityLogEntry -> entityLogEntry.setNumberOfComments(commentCountPerAnnotationId.getOrDefault(entityLogEntry.getId(), 0)));
@ -62,7 +94,7 @@ public class EntityLogService {
filteredEntityLogRequest.setSpecifiedDate(OffsetDateTime.MIN);
}
var entityLog = getEntityLog(dossierId, fileId, filteredEntityLogRequest.getExcludedTypes());
var entityLog = getEntityLog(dossierId, fileId, filteredEntityLogRequest.getExcludedTypes(), false);
var entityLogEntry = entityLog.getEntityLogEntry();
Iterator<EntityLogEntry> it = entityLogEntry.iterator();
@ -92,4 +124,184 @@ public class EntityLogService {
return entityLog;
}
private List<UnprocessedManualEntity> getUnprocessedManualEntities(String dossierId, String fileId, ManualRedactions manualRedactions) {
DossierEntity dossier = dossierService.getDossierById(dossierId);
return unprocessedManualEntityClient.mergeUnprocessedManualEntities(fileId, dossierId, dossier.getDossierTemplateId(), manualRedactions);
}
public void mergeEntityLog(ManualRedactions manualRedactions, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog) {
log.info("Merging EntityLog");
mergeManualRedactionEntries(manualRedactions.getEntriesToAdd(), unprocessedManualEntities, entityLog);
mergeIdsToRemove(manualRedactions.getIdsToRemove(), entityLog);
mergeResizeRedactions(manualRedactions.getResizeRedactions(), entityLog);
mergeLegalBasisChanges(manualRedactions.getLegalBasisChanges(), entityLog);
mergeRecategorizations(manualRedactions.getRecategorizations(), entityLog);
mergeForceRedactions(manualRedactions.getForceRedactions(), entityLog);
log.info("EntityLog merged successfully!");
}
private void mergeManualRedactionEntries(Set<ManualRedactionEntry> manualRedactionEntries, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog) {
manualRedactionEntries.forEach(manualRedactionEntry -> {
UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream()
.filter(manualEntity -> manualEntity.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
.findFirst().orElseThrow(() -> new NotFoundException("Entry with annotationId " + manualRedactionEntry.getAnnotationId() + " not found"));
List<Change> changes = new ArrayList<>();
changes.add(Change.builder()
.analysisNumber(1)
.dateTime(OffsetDateTime.now())
.type(ChangeType.ADDED)
.build());
entityLog.getEntityLogEntry().add(EntityLogEntry.builder()
.id(manualRedactionEntry.getAnnotationId())
.type(manualRedactionEntry.getType())
.value(manualRedactionEntry.getValue())
.legalBasis(manualRedactionEntry.getLegalBasis())
.reason(manualRedactionEntry.getReason())
.entryType(isHint(manualRedactionEntry.getType()) ? EntryType.HINT : EntryType.ENTITY)
.imported(false)
.matchedRule("")
.section(manualRedactionEntry.getSection())
.color(unprocessedManualEntity.getColor())
.positions(unprocessedManualEntity.getPositions())
.textAfter(unprocessedManualEntity.getTextAfter())
.textBefore(unprocessedManualEntity.getTextBefore())
.startOffset(unprocessedManualEntity.getStartOffset())
.endOffset(unprocessedManualEntity.getEndOffset())
.containingNodeId(unprocessedManualEntity.getContainingNodeId())
.closestHeadline(unprocessedManualEntity.getClosestHeadline())
.imageHasTransparency(false)
.dictionaryEntry(manualRedactionEntry.isAddToDictionary())
.dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.excluded(false)
.changes(changes)
.manualChanges(List.of(ManualChange.builder()
.manualRedactionType(ManualRedactionType.ADD_LOCALLY)
.requestedDate(manualRedactionEntry.getRequestDate())
.processedDate(manualRedactionEntry.getProcessedDate())
.userId(manualRedactionEntry.getUser())
.propertyChanges(Map.of("value", manualRedactionEntry.getValue()))
.build()))
.engines(new HashSet<>())
.reference(new HashSet<>())
.importedRedactionIntersections(new HashSet<>())
.build());
});
}
private void mergeIdsToRemove(Set<IdRemoval> idRemovals, EntityLog entityLog) {
idRemovals.forEach(idRemoval -> {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setState(EntryState.REMOVED);
addChanges(entity.get().getChanges(), ChangeType.REMOVED);
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(idRemoval.getProcessedDate())
.userId(idRemoval.getUser()).build());
}
});
}
private void mergeResizeRedactions(Set<ManualResizeRedaction> manualResizeRedactions, EntityLog entityLog) {
manualResizeRedactions.forEach(manualResizeRedaction -> {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
var newPosition = manualResizeRedaction.getPositions().get(0);
entity.get().setPositions(List.of(new Position(
newPosition.getTopLeftX(),
newPosition.getTopLeftY(),
newPosition.getWidth(),
newPosition.getHeight(),
newPosition.getPage())));
addChanges(entity.get().getChanges(), ChangeType.CHANGED);
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RESIZE)
.requestedDate(manualResizeRedaction.getRequestDate())
.processedDate(manualResizeRedaction.getProcessedDate())
.userId(manualResizeRedaction.getUser()).build());
}
});
}
private void mergeLegalBasisChanges(Set<ManualLegalBasisChange> manualLegalBasisChanges, EntityLog entityLog) {
manualLegalBasisChanges.forEach(manualLegalBasisChange -> {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis());
addChanges(entity.get().getChanges(), ChangeType.CHANGED);
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(manualLegalBasisChange.getProcessedDate())
.propertyChanges(Map.of("value", manualLegalBasisChange.getValue()))
.userId(manualLegalBasisChange.getUser())
.build());
}
});
}
private void mergeRecategorizations(Set<ManualRecategorization> recategorizations, EntityLog entityLog) {
recategorizations.forEach(recategorization -> {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setType(recategorization.getType());
addChanges(entity.get().getChanges(), ChangeType.CHANGED);
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
.requestedDate(recategorization.getRequestDate())
.processedDate(recategorization.getProcessedDate())
.userId(recategorization.getUser())
.build());
}
});
}
private void mergeForceRedactions(Set<ManualForceRedaction> forceRedactions, EntityLog entityLog) {
forceRedactions.forEach(forceRedaction -> {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setLegalBasis(forceRedaction.getLegalBasis());
entity.get().setState(EntryState.APPLIED);
addChanges(entity.get().getChanges(), ChangeType.CHANGED);
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser())
.build());
}
});
}
private void addChanges(List<Change> changes, ChangeType changeType) {
if (!changes.isEmpty()) {
var lastChange = changes.get(changes.size() -1);
changes.add(Change.builder()
.analysisNumber(lastChange.getAnalysisNumber() + 1)
.dateTime(OffsetDateTime.now())
.type(changeType)
.build());
} else {
changes.add(Change.builder().analysisNumber(1).dateTime(OffsetDateTime.now()).type(changeType).build());
}
}
private boolean isHint(String manualRedactionEntryType) {
return !(manualRedactionEntryType.equals("manual"));
}
}

View File

@ -0,0 +1,260 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
public class EntityLogTest extends AbstractPersistenceServerServiceTest {
@Autowired
private FileTesterAndProvider fileTesterAndProvider;
@Autowired
private DossierTesterAndProvider dossierTesterAndProvider;
@Autowired
private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
@Autowired
private FileStatusService fileStatusService;
@Autowired
private FileManagementStorageService fileManagementStorageService;
@MockBean
private ManualRedactionProviderService manualRedactionProviderService;
@MockBean
private UnprocessedManualEntityClient unprocessedManualEntityClient;
@Autowired
private EntityLogService entityLogService;
@Test
public void testGetEntityLogWithUnprocessedRedactions() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String entryToAddId = UUID.randomUUID().toString();
String entryToRemoveId = UUID.randomUUID().toString();
String entryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, file.getId());
UnprocessedManualEntity unprocessedManualEntity = UnprocessedManualEntity.builder()
.textBefore("textBefore")
.textAfter("textAfter")
.containingNodeId(List.of(0, 0))
.endOffset(10)
.startOffset(1)
.annotationId(entryToAddId)
.closestHeadline("closestHeadline")
.color(new float[]{0,0,0})
.build();
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
when(manualRedactionProviderService.getManualRedactions(file.getId(), true)).thenReturn(manualRedactions);
when(unprocessedManualEntityClient.mergeUnprocessedManualEntities(any(), any(), any(), any())).thenReturn(List.of(unprocessedManualEntity));
EntityLog response = entityLogService.getEntityLog(dossier.getId(), file.getId(), Collections.emptyList(), true);
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());
var optionalEntityLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToAddId)).findFirst();
assertTrue(optionalEntityLogEntry.isPresent());
var entityLogEntry = optionalEntityLogEntry.get();
assertEquals(entityLogEntry.getType(), "manual");
assertEquals(entityLogEntry.getEntryType(), EntryType.ENTITY);
assertNull(entityLogEntry.getState());
assertEquals(entityLogEntry.getValue(), "Test");
assertEquals(entityLogEntry.getReason(), "Reason");
assertEquals(entityLogEntry.getTextAfter(), "textAfter");
assertEquals(entityLogEntry.getTextBefore(), "textBefore");
assertEquals(entityLogEntry.getStartOffset(), 1);
assertEquals(entityLogEntry.getEndOffset(), 10);
assertEquals(entityLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.ADD_LOCALLY);
var optionalRemoveEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRemoveId)).findFirst();
assertTrue(optionalRemoveEntryLogEntry.isPresent());
var removeEntryLogEntry = optionalRemoveEntryLogEntry.get();
assertEquals(removeEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(removeEntryLogEntry.getState(), EntryState.REMOVED);
assertEquals(removeEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.REMOVE_LOCALLY);
assertEquals(removeEntryLogEntry.getChanges().get(0).getType(), ChangeType.REMOVED);
var optionalResizeEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)).findFirst();
assertTrue(optionalResizeEntryLogEntry.isPresent());
var resizeEntryLogEntry = optionalResizeEntryLogEntry.get();
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[0], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[1], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[2], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[3], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getPageNumber(), 1);
assertEquals(resizeEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(resizeEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(resizeEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertEquals(resizeEntryLogEntry.getChanges().get(0).getType(), ChangeType.CHANGED);
var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId)).findFirst();
assertTrue(optionalLegalBasisEntryLogEntry.isPresent());
var legalBasisEntryLogEntry = optionalLegalBasisEntryLogEntry.get();
assertEquals(legalBasisEntryLogEntry.getLegalBasis(), "New legal basis");
assertEquals(legalBasisEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(legalBasisEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(legalBasisEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.LEGAL_BASIS_CHANGE);
assertEquals(legalBasisEntryLogEntry.getChanges().get(0).getType(), ChangeType.CHANGED);
var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId)).findFirst();
assertTrue(optionalForceRedactionEntryLogEntry.isPresent());
var forceRedactionEntryLogEntry = optionalForceRedactionEntryLogEntry.get();
assertEquals(forceRedactionEntryLogEntry.getLegalBasis(), "Force");
assertEquals(forceRedactionEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(forceRedactionEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(forceRedactionEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.FORCE_REDACT);
assertEquals(forceRedactionEntryLogEntry.getChanges().get(0).getType(), ChangeType.CHANGED);
}
private EntityLog provideEntityLog(String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId) {
List<Position> positions = new ArrayList<>();
positions.add(new Position(1, 1, 1, 1, 1));
return new EntityLog(1,
1,
List.of(EntityLogEntry.builder()
.id(entryToRemoveId)
.type("manual")
.value("Luke Skywalker")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.build(),
EntityLogEntry.builder()
.id(entryToResizeId)
.type("manual")
.value("Darth Vader")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(entryLegalBasisId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(forceRedactionId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build()),
null,
0,
0,
0,
0);
}
private ManualRedactions provideManualRedactions(String entryToAddId, String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId, String fileId) {
List<Rectangle> positions = new ArrayList<>();
positions.add(new Rectangle(2, 2, 2, 2, 1));
return ManualRedactions.builder()
.entriesToAdd(Set.of(
ManualRedactionEntry.builder()
.positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
.annotationId(entryToAddId)
.value("Test")
.reason("Reason")
.addToDictionary(false)
.addToDossierDictionary(false)
.fileId(fileId)
.requestDate(OffsetDateTime.now())
.type("manual")
.build()))
.idsToRemove(Set.of(
IdRemoval.builder()
.annotationId(entryToRemoveId)
.build()
))
.resizeRedactions(Set.of(
ManualResizeRedaction.builder()
.fileId(fileId)
.value("Random")
.annotationId(entryToResizeId)
.positions(positions)
.build()
))
.legalBasisChanges(Set.of(
ManualLegalBasisChange.builder()
.annotationId(entryLegalBasisId)
.value("Random")
.legalBasis("New legal basis")
.build()
))
.forceRedactions(Set.of(
ManualForceRedaction.builder()
.annotationId(forceRedactionId)
.fileId(fileId)
.legalBasis("Force")
.build()
))
.build();
}
}

View File

@ -10,10 +10,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.assertj.core.util.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
@ -91,7 +94,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
@Autowired
private InternalDictionaryClient internalDictionaryClient;
@Autowired
@MockBean
private EntityLogService entityLogService;
@Autowired
@ -103,6 +106,11 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
@Autowired
private FileProcessingClient fileProcessingClient;
@BeforeEach
public void setup() {
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(new EntityLog(1, 1, Lists.newArrayList(), null, 0, 0, 0, 0));
}
@Test
public void testRemoveToDossierTemplateWithDossierDictionaryOnlyTrue() {

View File

pan>
@ -1,40 +1,13 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient;
import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.*;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Lists;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
@ -51,7 +24,11 @@ import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.StatementCallback;
import org.springframework.jdbc.datasource.SingleConnectionDataSource;
@ -67,13 +44,69 @@ import org.springframework.security.web.SecurityFilterChain;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient;
import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.IndexInformationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationPreferencesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ForceRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.LegalBasisChangeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RecategorizationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.AuthDetails;
import com.knecon.fforesight.tenantcommons.model.DatabaseConnection;
import com.knecon.fforesight.tenantcommons.model.S3StorageConnection;
import com.knecon.fforesight.tenantcommons.model.SearchConnection;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import static org.mockito.Mockito.when;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ExtendWith(SpringExtension.class)
@ -92,10 +125,10 @@ public abstract class AbstractPersistenceServerServiceTest {
@MockBean
protected SearchClient searchClient;
@MockBean
protected EntityLogService entityLogService;
@MockBean
protected PDFTronClient pdfTronRedactionClient;
@Autowired
protected EntityLogService entityLogService;
@Autowired
protected ApplicationConfigClient appConfigClient;
@Autowired
protected StorageService storageService;
@ -190,8 +223,7 @@ public abstract class AbstractPersistenceServerServiceTest {
var allRoles = ApplicationRoles.ROLE_DATA.entrySet().stream().flatMap(entry -> entry.getValue().stream()).collect(Collectors.toSet());
allRoles.addAll(ApplicationRoles.UNMAPPED_ACTION_ROLES);
allRoles.addAll(ApplicationRoles.ROLE_DATA.keySet());
var rolesArray = allRoles.toArray(new String[0]);
return rolesArray;
return allRoles.toArray(new String[0]);
}
@ -245,7 +277,6 @@ public abstract class AbstractPersistenceServerServiceTest {
// doNothing().when(pdfTronRedactionClient).testDigitalCurrentSignature(Mockito.any());
when(amqpAdmin.getQueueInfo(Mockito.any())).thenReturn(null);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(new EntityLog(1, 1, Lists.newArrayList(), null, 0, 0, 0, 0));
when(redactionClient.testRules(Mockito.any())).thenReturn(DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(Collections.emptyList()).build());
}

View File

@ -3,6 +3,5 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model;
public enum MessageType {
ANALYSE,
REANALYSE,
SURROUNDING_TEXT
REANALYSE
}

View File

@ -31,7 +31,7 @@
</modules>
<properties>
<redaction-service.version>4.126.0</redaction-service.version>
<redaction-service.version>4.157.0</redaction-service.version>
<search-service.version>2.71.0</search-service.version>
<pdftron-redaction-service.version>4.29.0</pdftron-redaction-service.version>
<redaction-report-service.version>4.13.0</redaction-report-service.version>