RED-9140 - Add more information to changes
This commit is contained in:
Andrei Isvoran
parent
9e45864b35
commit
a58bcedccf
@ -4,7 +4,7 @@ plugins {
|
||||
}
|
||||
|
||||
description = "redaction-service-api-v1"
|
||||
val persistenceServiceVersion = "2.439.0"
|
||||
val persistenceServiceVersion = "2.467.0"
|
||||
|
||||
dependencies {
|
||||
implementation("org.springframework:spring-web:6.0.12")
|
||||
|
||||
@ -16,7 +16,7 @@ val layoutParserVersion = "0.141.0"
|
||||
val jacksonVersion = "2.15.2"
|
||||
val droolsVersion = "9.44.0.Final"
|
||||
val pdfBoxVersion = "3.0.0"
|
||||
val persistenceServiceVersion = "2.444.0"
|
||||
val persistenceServiceVersion = "2.467.0"
|
||||
val springBootStarterVersion = "3.1.5"
|
||||
val springCloudVersion = "4.0.4"
|
||||
val testContainersVersion = "1.19.7"
|
||||
|
||||
@ -18,7 +18,8 @@ public class MigrationMapper {
|
||||
|
||||
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
|
||||
toEntityLogType(change.getType()),
|
||||
change.getDateTime());
|
||||
change.getDateTime(),
|
||||
Collections.emptyMap());
|
||||
}
|
||||
|
||||
|
||||
@ -28,7 +29,8 @@ public class MigrationMapper {
|
||||
manualChange.getProcessedDate(),
|
||||
manualChange.getRequestedDate(),
|
||||
manualChange.getUserId(),
|
||||
manualChange.getPropertyChanges());
|
||||
manualChange.getPropertyChanges(),
|
||||
0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -198,7 +198,7 @@ public final class MigrationEntity {
|
||||
throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
|
||||
}
|
||||
|
||||
entityLogEntry.setManualChanges(ManualChangeFactory.toLocalManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), true));
|
||||
entityLogEntry.setManualChanges(ManualChangeFactory.toLocalManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), true, 0));
|
||||
entityLogEntry.setColor(redactionLogEntry.getColor());
|
||||
entityLogEntry.setChanges(redactionLogEntry.getChanges()
|
||||
.stream()
|
||||
|
||||
@ -2,7 +2,9 @@ package com.iqser.red.service.redaction.v1.server.service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
@ -13,6 +15,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.AccessLevel;
|
||||
@ -31,7 +38,7 @@ public class EntityChangeLogService {
|
||||
|
||||
var now = OffsetDateTime.now();
|
||||
if (previousEntityLogEntries.isEmpty()) {
|
||||
newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)));
|
||||
newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())));
|
||||
return new EntryChanges(newEntityLogEntries, new ArrayList<>());
|
||||
}
|
||||
|
||||
@ -42,7 +49,7 @@ public class EntityChangeLogService {
|
||||
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
|
||||
.findAny();
|
||||
if (optionalPreviousEntity.isEmpty()) {
|
||||
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
|
||||
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()));
|
||||
toInsert.add(entityLogEntry);
|
||||
continue;
|
||||
}
|
||||
@ -51,10 +58,9 @@ public class EntityChangeLogService {
|
||||
entityLogEntry.getChanges().addAll(previousEntity.getChanges());
|
||||
|
||||
if (!previousEntity.equals(entityLogEntry)) {
|
||||
if(!previousEntity.getState().equals(entityLogEntry.getState())) {
|
||||
ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
|
||||
entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
|
||||
}
|
||||
List<PropertyChange> propertyChanges = EntityLogEntryDiffChecker.compareEntityLogEntries(previousEntity, entityLogEntry);
|
||||
Change change = calculateChange(previousEntity, entityLogEntry, propertyChanges, analysisNumber);
|
||||
entityLogEntry.getChanges().add(change);
|
||||
toUpdate.add(entityLogEntry);
|
||||
}
|
||||
}
|
||||
@ -77,18 +83,43 @@ public class EntityChangeLogService {
|
||||
.toList();
|
||||
removedEntries.stream()
|
||||
.filter(entry -> !entry.getState().equals(EntryState.REMOVED))
|
||||
.peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
|
||||
.peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now, Collections.emptyMap())))
|
||||
.forEach(entry -> entry.setState(EntryState.REMOVED));
|
||||
newEntityLogEntries.addAll(removedEntries);
|
||||
return removedEntries;
|
||||
}
|
||||
|
||||
|
||||
private Change calculateChange(EntityLogEntry previousEntry, EntityLogEntry currentEntry, List<PropertyChange> propertyChanges, int analysisNumber) {
|
||||
|
||||
ManualChange previousLastManualChange;
|
||||
ManualChange currentLastManualChange;
|
||||
|
||||
if (previousEntry.getManualChanges().isEmpty() && currentEntry.getManualChanges().isEmpty()) {
|
||||
return systemChange(propertyChanges, analysisNumber, previousEntry.getState(), currentEntry.getState());
|
||||
}
|
||||
|
||||
if (previousEntry.getManualChanges().isEmpty() && !currentEntry.getManualChanges().isEmpty()) {
|
||||
currentLastManualChange = currentEntry.getManualChanges()
|
||||
.get(currentEntry.getManualChanges().size() - 1);
|
||||
return manualChange(propertyChanges, analysisNumber, currentLastManualChange);
|
||||
}
|
||||
|
||||
previousLastManualChange = previousEntry.getManualChanges()
|
||||
.get(previousEntry.getManualChanges().size() - 1);
|
||||
currentLastManualChange = currentEntry.getManualChanges()
|
||||
.get(currentEntry.getManualChanges().size() - 1);
|
||||
|
||||
if (Objects.equals(previousLastManualChange, currentLastManualChange)) {
|
||||
return systemChange(propertyChanges, analysisNumber, previousEntry.getState(), currentEntry.getState());
|
||||
} else {
|
||||
return manualChange(propertyChanges, analysisNumber, currentLastManualChange);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
|
||||
|
||||
if (state.equals(previousState)) {
|
||||
throw new IllegalArgumentException("States are equal, can't calculate ChangeType.");
|
||||
}
|
||||
if (!isRemoved(previousState) && isRemoved(state)) {
|
||||
return ChangeType.REMOVED;
|
||||
}
|
||||
@ -99,12 +130,40 @@ public class EntityChangeLogService {
|
||||
}
|
||||
|
||||
|
||||
private Change systemChange(List<PropertyChange> propertyChanges, int analysisNumber, EntryState previousState, EntryState currentState) {
|
||||
|
||||
return ChangeFactory.toChange(calculateChangeType(currentState, previousState), OffsetDateTime.now(), analysisNumber, propertyChanges.toArray(new PropertyChange[0]));
|
||||
}
|
||||
|
||||
|
||||
private Change manualChange(List<PropertyChange> propertyChanges, int analysisNumber, ManualChange manualChange) {
|
||||
|
||||
return ChangeFactory.toChange(convertToChangeType(manualChange.getManualRedactionType()),
|
||||
manualChange.getRequestedDate(),
|
||||
analysisNumber,
|
||||
propertyChanges.toArray(new PropertyChange[0]));
|
||||
}
|
||||
|
||||
|
||||
private static boolean isRemoved(EntryState state) {
|
||||
|
||||
return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
|
||||
}
|
||||
|
||||
|
||||
public ChangeType convertToChangeType(ManualRedactionType manualRedactionType) {
|
||||
|
||||
return switch (manualRedactionType) {
|
||||
case ADD_LOCALLY, ADD, ADD_TO_DICTIONARY -> ChangeType.ADDED;
|
||||
case FORCE_REDACT, FORCE_HINT, FORCE -> ChangeType.FORCE_REDACT;
|
||||
case REMOVE_LOCALLY, REMOVE, REMOVE_FROM_DICTIONARY -> ChangeType.REMOVED;
|
||||
case RECATEGORIZE, RECATEGORIZE_IN_DICTIONARY -> ChangeType.RECATEGORIZE;
|
||||
case LEGAL_BASIS_CHANGE -> ChangeType.LEGAL_BASIS_CHANGE;
|
||||
case RESIZE, RESIZE_IN_DICTIONARY -> ChangeType.RESIZED;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public record EntryChanges(List<EntityLogEntry> inserted, List<EntityLogEntry> updated) {
|
||||
|
||||
}
|
||||
|
||||
@ -20,9 +20,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
||||
@ -39,6 +41,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -56,6 +59,7 @@ public class EntityLogCreatorService {
|
||||
LegalBasisClient legalBasisClient;
|
||||
EntityChangeLogService entityChangeLogService;
|
||||
RedactionStorageService redactionStorageService;
|
||||
EntityLogMongoService entityLogMongoService;
|
||||
|
||||
|
||||
private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
|
||||
@ -72,7 +76,7 @@ public class EntityLogCreatorService {
|
||||
DictionaryVersion dictionaryVersion,
|
||||
long rulesVersion) {
|
||||
|
||||
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
|
||||
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities, analyzeRequest.getAnalysisNumber());
|
||||
|
||||
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||
|
||||
@ -129,7 +133,7 @@ public class EntityLogCreatorService {
|
||||
Set<Integer> sectionsToReanalyseIds,
|
||||
DictionaryVersion dictionaryVersion) {
|
||||
|
||||
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
|
||||
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
|
||||
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
|
||||
.get(0)))
|
||||
.collect(Collectors.toList());
|
||||
@ -144,7 +148,7 @@ public class EntityLogCreatorService {
|
||||
}
|
||||
|
||||
|
||||
private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
|
||||
private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries, int analysisNumber) {
|
||||
|
||||
String dossierTemplateId = analyzeRequest.getDossierTemplateId();
|
||||
|
||||
@ -155,31 +159,34 @@ public class EntityLogCreatorService {
|
||||
.filter(entity -> !entity.getValue().isEmpty())
|
||||
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
|
||||
.filter(entity -> !entity.removed())
|
||||
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
|
||||
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
|
||||
|
||||
document.streamAllImages()
|
||||
.filter(entity -> !entity.removed())
|
||||
.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
|
||||
.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
|
||||
|
||||
notFoundPrecursorEntries.stream()
|
||||
.filter(entity -> !entity.removed())
|
||||
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity)));
|
||||
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
|
||||
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
|
||||
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) {
|
||||
|
||||
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
|
||||
|
||||
// split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
|
||||
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
|
||||
|
||||
EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
|
||||
|
||||
List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
|
||||
.stream()
|
||||
.map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
|
||||
.toList();
|
||||
|
||||
EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId);
|
||||
|
||||
// set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
|
||||
entityLogEntry.setId(positionOnPage.getId());
|
||||
entityLogEntry.setPositions(rectanglesPerLine);
|
||||
@ -190,10 +197,14 @@ public class EntityLogCreatorService {
|
||||
}
|
||||
|
||||
|
||||
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
|
||||
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) {
|
||||
|
||||
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
|
||||
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
|
||||
|
||||
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId());
|
||||
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
|
||||
|
||||
return EntityLogEntry.builder()
|
||||
.id(image.getId())
|
||||
.value(image.getValue())
|
||||
@ -209,7 +220,7 @@ public class EntityLogCreatorService {
|
||||
// .orElse(image.getParent().toString()))
|
||||
.orElse(this.buildSectionString(image.getParent())))
|
||||
.imageHasTransparency(image.isTransparent())
|
||||
.manualChanges(ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true))
|
||||
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
|
||||
.state(buildEntryState(image))
|
||||
.entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||
.engines(getEngines(null, image.getManualOverwrite()))
|
||||
@ -219,11 +230,14 @@ public class EntityLogCreatorService {
|
||||
}
|
||||
|
||||
|
||||
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
|
||||
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) {
|
||||
|
||||
String type = precursorEntity.getManualOverwrite().getType()
|
||||
.orElse(precursorEntity.getType());
|
||||
boolean isHint = isHint(precursorEntity.getEntityType());
|
||||
|
||||
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
|
||||
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
|
||||
|
||||
return EntityLogEntry.builder()
|
||||
.id(precursorEntity.getId())
|
||||
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
|
||||
@ -253,13 +267,13 @@ public class EntityLogCreatorService {
|
||||
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
|
||||
.imported(false)
|
||||
.reference(Collections.emptySet())
|
||||
.manualChanges(ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true))
|
||||
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
|
||||
.paragraphPageIdx(-1)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private EntityLogEntry createEntityLogEntry(TextEntity entity) {
|
||||
private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) {
|
||||
|
||||
Set<String> referenceIds = new HashSet<>();
|
||||
entity.references()
|
||||
@ -270,6 +284,9 @@ public class EntityLogCreatorService {
|
||||
|
||||
EntryType entryType = buildEntryType(entity);
|
||||
|
||||
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id);
|
||||
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
|
||||
|
||||
return EntityLogEntry.builder()
|
||||
.reason(entity.buildReasonWithManualChangeDescriptions())
|
||||
.legalBasis(entity.legalBasis())
|
||||
@ -297,7 +314,7 @@ public class EntityLogCreatorService {
|
||||
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
|
||||
.imported(false)
|
||||
.reference(referenceIds)
|
||||
.manualChanges(ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true))
|
||||
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
|
||||
.state(buildEntryState(entity))
|
||||
.entryType(entryType)
|
||||
.paragraphPageIdx(determinePageParagraphIndex(entity, entryType))
|
||||
@ -403,4 +420,17 @@ public class EntityLogCreatorService {
|
||||
return node.getType().toString() + ": " + node.getTextBlock().buildSummary();
|
||||
}
|
||||
|
||||
|
||||
public List<ManualChange> getManualChangesByEntityLogId(String dossierId, String fileId, String id) {
|
||||
|
||||
List<ManualChange> manualChanges = new ArrayList<>();
|
||||
List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id));
|
||||
|
||||
for (EntityLogEntry entry : entityLogEntries) {
|
||||
manualChanges.addAll(entry.getManualChanges());
|
||||
}
|
||||
|
||||
return manualChanges;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
|
||||
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
@ -96,7 +97,7 @@ public class NotFoundImportedEntitiesService {
|
||||
|
||||
if (entityLogEntry.getState() != EntryState.REMOVED) {
|
||||
entityLogEntry.setState(EntryState.REMOVED);
|
||||
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
|
||||
entityLogEntry.getChanges().add(ChangeFactory.toChange(ChangeType.REMOVED, OffsetDateTime.now(), analysisNumber));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,45 @@
|
||||
package com.iqser.red.service.redaction.v1.server.utils;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@UtilityClass
|
||||
@Slf4j
|
||||
@SuppressWarnings("PMD") // Needed in order to be able to do setAccessible(true) for field comparison
|
||||
public class EntityLogEntryDiffChecker {
|
||||
|
||||
public List<PropertyChange> compareEntityLogEntries(EntityLogEntry previousEntityLogEntry, EntityLogEntry currentEntityLogEntry) {
|
||||
|
||||
List<PropertyChange> changes = new ArrayList<>();
|
||||
Field[] fields = EntityLogEntry.class.getDeclaredFields();
|
||||
|
||||
for (Field field : fields) {
|
||||
field.setAccessible(true);
|
||||
try {
|
||||
Object oldValue = field.get(previousEntityLogEntry);
|
||||
Object newValue = field.get(currentEntityLogEntry);
|
||||
|
||||
if (!Objects.equals(oldValue, newValue)) {
|
||||
changes.add(PropertyChange.builder()
|
||||
.property(field.getName())
|
||||
.oldValue(oldValue != null ? oldValue.toString() : "")
|
||||
.newValue(newValue != null ? newValue.toString() : "")
|
||||
.build());
|
||||
}
|
||||
} catch (IllegalAccessException e) {
|
||||
log.warn("Failed to access field: {}", field.getName());
|
||||
}
|
||||
}
|
||||
|
||||
return changes;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
package com.iqser.red.service.redaction.v1.server.utils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@UtilityClass
|
||||
@Slf4j
|
||||
public class ManualChangesUtils {
|
||||
|
||||
// there is a very slight variation of time when comparing the manual changes request Date, under 1ms.
|
||||
public static final Duration TOLERANCE = Duration.ofMillis(1);
|
||||
|
||||
|
||||
public List<ManualChange> mergeManualChanges(List<ManualChange> existingChanges, List<ManualChange> allChanges) {
|
||||
|
||||
List<ManualChange> mergedChanges = new ArrayList<>(existingChanges);
|
||||
|
||||
for (ManualChange manualChange : allChanges) {
|
||||
|
||||
var existingChangeOpt = mergedChanges.stream()
|
||||
.filter(existingChange -> areChangesEquivalent(existingChange, manualChange))
|
||||
.findFirst();
|
||||
|
||||
if (existingChangeOpt.isEmpty()) {
|
||||
mergedChanges.add(manualChange);
|
||||
}
|
||||
}
|
||||
|
||||
return mergedChanges;
|
||||
}
|
||||
|
||||
|
||||
private boolean areChangesEquivalent(ManualChange change1, ManualChange change2) {
|
||||
|
||||
return change1.getManualRedactionType() == change2.getManualRedactionType() && areDatesEquivalent(change1.getRequestedDate(), change2.getRequestedDate()) && Objects.equals(
|
||||
change1.getUserId(),
|
||||
change2.getUserId());
|
||||
}
|
||||
|
||||
|
||||
private boolean areDatesEquivalent(OffsetDateTime date1, OffsetDateTime date2) {
|
||||
|
||||
return Duration.between(date1, date2).abs().compareTo(TOLERANCE) <= 0;
|
||||
}
|
||||
|
||||
}
|
||||
@ -47,6 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
@ -1317,7 +1318,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
||||
.findFirst()
|
||||
.get();
|
||||
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder()
|
||||
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
|
||||
.annotationId("3029651d0842a625f2d23f8375c23600")
|
||||
@ -1840,6 +1840,283 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testResizeWithChanges() {
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var libraryOutlook = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().equals("library@outlook.com"))
|
||||
.findFirst();
|
||||
assertTrue(libraryOutlook.isPresent());
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder()
|
||||
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
|
||||
.addToAllDossiers(false)
|
||||
.updateDictionary(false)
|
||||
.user("user")
|
||||
.fileId(TEST_FILE_ID)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.annotationId(libraryOutlook.get().getId())
|
||||
.positions(List.of(Rectangle.builder()
|
||||
.topLeftX(159.364f)
|
||||
.topLeftY(303.364f)
|
||||
.width(115.68f)
|
||||
.height(15.408f)
|
||||
.page(4)
|
||||
.build()))
|
||||
.value("in library@outlook.com")
|
||||
.build()))
|
||||
.build());
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var inLibraryOutlookOpt = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().equals("in library@outlook.com"))
|
||||
.findFirst();
|
||||
assertTrue(inLibraryOutlookOpt.isPresent());
|
||||
var inLibraryOutlook = inLibraryOutlookOpt.get();
|
||||
assertEquals(inLibraryOutlook.getChanges().size(), 2);
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(0).getType(), ChangeType.ADDED);
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getType(), ChangeType.RESIZED);
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("reason"), "Found by Email Regex -> Found by Email Regex, resized by manual override");
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("startOffset"), "3793 -> 3790");
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("legalBasis"), "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data) -> Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("positions"), "[[171.748, 305.0, 103.296036, 12.642, 4]] -> [[159.364, 303.364, 115.68, 15.408, 4]]");
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("textBefore"), "irure dolor in -> aute irure dolor ");
|
||||
assertEquals(inLibraryOutlook.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("value"), "library@outlook.com -> in library@outlook.com");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAddWithChanges() {
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
String manualAddId = UUID.randomUUID().toString();
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder()
|
||||
.entriesToAdd(Set.of(ManualRedactionEntry.builder()
|
||||
.annotationId(manualAddId)
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.fileId(TEST_FILE_ID)
|
||||
.user("user")
|
||||
.value("not in Dictionary")
|
||||
.section(null)
|
||||
.reason("(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)")
|
||||
.addToDossierDictionary(false)
|
||||
.addToDictionary(false)
|
||||
.legalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002")
|
||||
.rectangle(false)
|
||||
.positions(List.of(Rectangle.builder()
|
||||
.topLeftX(270.844f)
|
||||
.topLeftY(238.364f)
|
||||
.width(81.876f)
|
||||
.height(15.408f)
|
||||
.page(1)
|
||||
.build()))
|
||||
.type("manual")
|
||||
.build()))
|
||||
.build());
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var notInDictionaryOpt = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getId().equals(manualAddId))
|
||||
.findFirst();
|
||||
assertTrue(notInDictionaryOpt.isPresent());
|
||||
var notInDictionary = notInDictionaryOpt.get();
|
||||
assertEquals(notInDictionary.getChanges().size(), 1);
|
||||
assertEquals(notInDictionary.getChanges()
|
||||
.get(0).getType(), ChangeType.ADDED);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testRemovalWithChanges() {
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var davidKsenia = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().equals("David Ksenia"))
|
||||
.findFirst();
|
||||
assertTrue(davidKsenia.isPresent());
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder()
|
||||
.idsToRemove(Set.of(IdRemoval.builder()
|
||||
.fileId(TEST_FILE_ID)
|
||||
.user("user")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.annotationId(davidKsenia.get().getId())
|
||||
.removeFromDictionary(false)
|
||||
.removeFromAllDossiers(false)
|
||||
.build()))
|
||||
.build());
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var davidKseniaOpt = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getId().equals(davidKsenia.get().getId()))
|
||||
.findFirst();
|
||||
assertTrue(davidKseniaOpt.isPresent());
|
||||
var responseDavidKsenia = davidKseniaOpt.get();
|
||||
assertEquals(responseDavidKsenia.getChanges().size(), 2);
|
||||
assertEquals(responseDavidKsenia.getState(), EntryState.IGNORED);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(0).getType(), ChangeType.ADDED);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getType(), ChangeType.REMOVED);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("reason"), "No vertebrate found -> removed by manual override");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("matchedRule"), "CBI.3.2 -> ");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("state"), "SKIPPED -> IGNORED");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testRecatgeorizeWithChanges() {
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var davidKsenia = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().equals("David Ksenia"))
|
||||
.findFirst();
|
||||
assertTrue(davidKsenia.isPresent());
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder()
|
||||
.recategorizations(Set.of(ManualRecategorization.builder()
|
||||
.fileId(TEST_FILE_ID)
|
||||
.user("user")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.annotationId(davidKsenia.get().getId())
|
||||
.legalBasis("new legal basis")
|
||||
.addToAllDossiers(false)
|
||||
.addToDictionary(false)
|
||||
.type("PII")
|
||||
.section(null)
|
||||
.build()))
|
||||
.build());
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var davidKseniaOpt = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getId().equals(davidKsenia.get().getId()))
|
||||
.findFirst();
|
||||
assertTrue(davidKseniaOpt.isPresent());
|
||||
var responseDavidKsenia = davidKseniaOpt.get();
|
||||
assertEquals(responseDavidKsenia.getChanges().size(), 2);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(0).getType(), ChangeType.ADDED);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getType(), ChangeType.RECATEGORIZE);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("reason"), "No vertebrate found -> Recategorized entities are applied by default., recategorized by manual override");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("matchedRule"), "CBI.3.2 -> MAN.3.3");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("legalBasis"), " -> new legal basis");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("state"), "SKIPPED -> APPLIED");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("type"), "CBI_author -> PII");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testForceWithChanges() {
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var davidKsenia = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().equals("David Ksenia"))
|
||||
.findFirst();
|
||||
assertTrue(davidKsenia.isPresent());
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder()
|
||||
.forceRedactions(Set.of(ManualForceRedaction.builder()
|
||||
.fileId(TEST_FILE_ID)
|
||||
.user("user")
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.annotationId(davidKsenia.get().getId())
|
||||
.legalBasis("new legal basis")
|
||||
.build()))
|
||||
.build());
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var davidKseniaOpt = entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(e -> e.getId().equals(davidKsenia.get().getId()))
|
||||
.findFirst();
|
||||
assertTrue(davidKseniaOpt.isPresent());
|
||||
var responseDavidKsenia = davidKseniaOpt.get();
|
||||
assertEquals(responseDavidKsenia.getChanges().size(), 2);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(0).getType(), ChangeType.ADDED);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getType(), ChangeType.FORCE_REDACT);
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("reason"), "No vertebrate found -> No vertebrate found, forced by manual override");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("legalBasis"), " -> new legal basis");
|
||||
assertEquals(responseDavidKsenia.getChanges()
|
||||
.get(1).getPropertyChanges()
|
||||
.get("state"), "SKIPPED -> APPLIED");
|
||||
}
|
||||
|
||||
|
||||
private IdRemoval getIdRemoval(String id) {
|
||||
|
||||
return IdRemoval.builder()
|
||||
|
||||
@ -10,14 +10,19 @@ import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.google.common.collect.Sets;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
||||
@ -25,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
|
||||
|
||||
public class ManualChangesIntegrationTest extends RulesIntegrationTest {
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user