RED-8043 - Calculate surrounding text for unprocessed manual resize
This commit is contained in:
parent
7b5c0026f9
commit
b42fae4054
@ -0,0 +1,17 @@
|
|||||||
|
package com.iqser.red.service.redaction.v1.server.model;
|
||||||
|
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
public class ClosestEntity {
|
||||||
|
|
||||||
|
private double distance;
|
||||||
|
private TextRange textRange;
|
||||||
|
|
||||||
|
}
|
||||||
@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.model;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.PriorityQueue;
|
import java.util.PriorityQueue;
|
||||||
|
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
@ -66,6 +68,26 @@ public class ManualEntity implements IEntity {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
|
||||||
|
|
||||||
|
List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
|
||||||
|
EntityType entityType = getEntityType(entityLogEntry.getEntryType());
|
||||||
|
return ManualEntity.builder()
|
||||||
|
.id(entityLogEntry.getId())
|
||||||
|
.value(entityLogEntry.getValue())
|
||||||
|
.entityPosition(rectangleWithPages)
|
||||||
|
.ruleIdentifier(entityLogEntry.getMatchedRule())
|
||||||
|
.reason(entityLogEntry.getReason())
|
||||||
|
.legalBasis(entityLogEntry.getLegalBasis())
|
||||||
|
.type(entityLogEntry.getType())
|
||||||
|
.section(entityLogEntry.getSection())
|
||||||
|
.entityType(entityType)
|
||||||
|
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
|
||||||
|
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
|
||||||
|
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public TextRange getTextRange() {
|
public TextRange getTextRange() {
|
||||||
@ -80,6 +102,27 @@ public class ManualEntity implements IEntity {
|
|||||||
return getManualOverwrite().getType().orElse(type);
|
return getManualOverwrite().getType().orElse(type);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static EntityType getEntityType(EntryType entryType) {
|
||||||
|
|
||||||
|
switch (entryType) {
|
||||||
|
case FALSE_RECOMMENDATION -> {
|
||||||
|
return EntityType.FALSE_RECOMMENDATION;
|
||||||
|
}
|
||||||
|
case FALSE_POSITIVE -> {
|
||||||
|
return EntityType.FALSE_POSITIVE;
|
||||||
|
}
|
||||||
|
case HINT -> {
|
||||||
|
return EntityType.HINT;
|
||||||
|
}
|
||||||
|
case RECOMMENDATION -> {
|
||||||
|
return EntityType.RECOMMENDATION;
|
||||||
|
}
|
||||||
|
default -> {
|
||||||
|
return EntityType.ENTITY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private EntityType getEntityType(boolean isHint) {
|
private EntityType getEntityType(boolean isHint) {
|
||||||
return isHint ? EntityType.HINT : EntityType.ENTITY;
|
return isHint ? EntityType.HINT : EntityType.ENTITY;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model;
|
|||||||
|
|
||||||
import java.awt.geom.Rectangle2D;
|
import java.awt.geom.Rectangle2D;
|
||||||
|
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||||
|
|
||||||
public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
||||||
@ -12,6 +13,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static RectangleWithPage fromEntityLogPosition(Position position) {
|
||||||
|
|
||||||
|
return new RectangleWithPage(position.getPageNumber(), toRectangle2D(position));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||||
|
|
||||||
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
||||||
@ -24,6 +31,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static Rectangle2D toRectangle2D(Position position) {
|
||||||
|
|
||||||
|
return new Rectangle2D.Float(position.x(), position.y(), position.w(), position.h());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||||
|
|
||||||
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
|
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
|
||||||
|
|||||||
@ -3,10 +3,17 @@ package com.iqser.red.service.redaction.v1.server.service;
|
|||||||
import java.awt.geom.Rectangle2D;
|
import java.awt.geom.Rectangle2D;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.NoSuchElementException;
|
import java.util.NoSuchElementException;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
@ -50,13 +57,46 @@ public class ManualChangesApplicationService {
|
|||||||
.findFirst()
|
.findFirst()
|
||||||
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
|
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
|
||||||
|
|
||||||
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList());
|
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
|
||||||
|
|
||||||
int newStartOffset;
|
String value = manualResizeRedaction.getValue();
|
||||||
if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
|
int newStartOffset = -1;
|
||||||
newStartOffset = entityToBeResized.getTextRange().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
|
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
|
||||||
} else {
|
ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
|
||||||
newStartOffset = entityToBeResized.getTextRange().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
|
|
||||||
|
// Loop through nodes starting from the deepest fully containing node all the way to the document node
|
||||||
|
while (node != null) {
|
||||||
|
if (node.containsString(value)) {
|
||||||
|
SearchImplementation searchImplementation = new SearchImplementation(value, false);
|
||||||
|
List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
|
||||||
|
|
||||||
|
for (TextRange textRange : textRanges) {
|
||||||
|
SemanticNode finalNode = node;
|
||||||
|
|
||||||
|
List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
|
||||||
|
.stream()
|
||||||
|
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
// If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
|
||||||
|
determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
|
||||||
|
|
||||||
|
// Remove all temp entities from the graph
|
||||||
|
tempEntities.forEach(TextEntity::removeFromGraph);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
|
||||||
|
if (node.hasParent()) {
|
||||||
|
node = node.getParent();
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (closestEntity.getTextRange() != null) {
|
||||||
|
newStartOffset = closestEntity.getTextRange().start();
|
||||||
}
|
}
|
||||||
|
|
||||||
// need to reinsert the entity, due to the boundary having changed.
|
// need to reinsert the entity, due to the boundary having changed.
|
||||||
@ -65,6 +105,31 @@ public class ManualChangesApplicationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction,
|
||||||
|
TextRange textRange,
|
||||||
|
List<TextEntity> tempEntities,
|
||||||
|
ClosestEntity closestEntity) {
|
||||||
|
|
||||||
|
double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage());
|
||||||
|
if (currentDistance < closestEntity.getDistance()) {
|
||||||
|
closestEntity.setDistance(currentDistance);
|
||||||
|
closestEntity.setTextRange(textRange);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static double calculateClosest(Rectangle position, List<PositionOnPage> positionOnPages) {
|
||||||
|
|
||||||
|
Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0);
|
||||||
|
double difference = 0;
|
||||||
|
difference += position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX());
|
||||||
|
difference += position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY());
|
||||||
|
difference += position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth());
|
||||||
|
difference += position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight());
|
||||||
|
return difference;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
|
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
|
||||||
|
|
||||||
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
|
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
|
||||||
@ -75,7 +140,9 @@ public class ManualChangesApplicationService {
|
|||||||
entityToBeResized.setPages(new HashSet<>());
|
entityToBeResized.setPages(new HashSet<>());
|
||||||
entityToBeResized.getTextRange().setStart(newStartOffset);
|
entityToBeResized.getTextRange().setStart(newStartOffset);
|
||||||
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||||
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
if (newStartOffset > -1) {
|
||||||
|
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -90,7 +157,7 @@ public class ManualChangesApplicationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
|
public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
|
||||||
|
|
||||||
return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
|
return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,8 +1,13 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.service;
|
package com.iqser.red.service.redaction.v1.server.service;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@ -10,20 +15,32 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
|
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
|
||||||
import com.iqser.red.service.redaction.v1.model.QueueNames;
|
import com.iqser.red.service.redaction.v1.model.QueueNames;
|
||||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
|
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
|
||||||
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
|
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||||
|
|
||||||
import io.micrometer.observation.annotation.Observed;
|
import io.micrometer.observation.annotation.Observed;
|
||||||
|
import jakarta.annotation.PostConstruct;
|
||||||
import lombok.AccessLevel;
|
import lombok.AccessLevel;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.experimental.FieldDefaults;
|
import lombok.experimental.FieldDefaults;
|
||||||
@ -32,28 +49,52 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||||
public class UnprocessedChangesService {
|
public class UnprocessedChangesService {
|
||||||
|
|
||||||
ManualEntityCreationService manualEntityCreationService;
|
private static final double THRESHOLD = 10;
|
||||||
RabbitTemplate rabbitTemplate;
|
|
||||||
ObservedStorageService observedStorageService;
|
final RabbitTemplate rabbitTemplate;
|
||||||
|
final ObservedStorageService observedStorageService;
|
||||||
|
final ManualRedactionEntryService manualRedactionEntryService;
|
||||||
|
final EntityFindingUtility entityFindingUtility;
|
||||||
|
final RedactionStorageService redactionStorageService;
|
||||||
|
final EntityEnrichmentService entityEnrichmentService;
|
||||||
|
EntityCreationService entityCreationService;
|
||||||
|
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
public void initEntityCreationService() {
|
||||||
|
|
||||||
|
entityCreationService = new EntityCreationService(entityEnrichmentService);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
|
@Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
|
||||||
public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
|
public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
|
||||||
|
|
||||||
List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
|
List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
|
||||||
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
|
||||||
Set<String> annotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
|
|
||||||
annotationIds.addAll(analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()));
|
|
||||||
|
|
||||||
List<ManualEntity> notFoundManualEntities = new ArrayList<>();
|
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||||
List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions());
|
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||||
if (!manualEntities.isEmpty()) {
|
|
||||||
notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
|
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
|
||||||
|
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
|
||||||
|
allAnnotationIds.addAll(resizeIds);
|
||||||
|
|
||||||
|
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
|
||||||
|
List<ManualEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
|
||||||
|
.stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList()
|
||||||
|
.stream().map(ManualEntity::fromEntityLogEntry).toList();
|
||||||
|
|
||||||
|
if (!manualResizeRedactions.isEmpty()) {
|
||||||
|
processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
|
||||||
|
document,
|
||||||
|
analyzeRequest.getDossierTemplateId());
|
||||||
|
|
||||||
document.getEntities().forEach(textEntity -> {
|
document.getEntities().forEach(textEntity -> {
|
||||||
Set<String> processedIds = new HashSet<>();
|
Set<String> processedIds = new HashSet<>();
|
||||||
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
|
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
|
||||||
@ -64,9 +105,9 @@ public class UnprocessedChangesService {
|
|||||||
List<Position> positions = positionsOnPerPage.getRectanglePerLine()
|
List<Position> positions = positionsOnPerPage.getRectanglePerLine()
|
||||||
.stream()
|
.stream()
|
||||||
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
|
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
|
||||||
.toList();
|
.collect(Collectors.toList());
|
||||||
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
||||||
.annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
|
.annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
|
||||||
.textBefore(textEntity.getTextBefore())
|
.textBefore(textEntity.getTextBefore())
|
||||||
.textAfter(textEntity.getTextAfter())
|
.textAfter(textEntity.getTextAfter())
|
||||||
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
|
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
|
||||||
@ -75,7 +116,140 @@ public class UnprocessedChangesService {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
notFoundManualRedactionEntries.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
|
||||||
|
|
||||||
|
rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
|
||||||
|
AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void processManualResizeRedactions(Document document,
|
||||||
|
List<ManualEntity> manualEntities,
|
||||||
|
List<UnprocessedManualEntity> unprocessedManualEntities,
|
||||||
|
List<ManualResizeRedaction> manualResizeRedactions) {
|
||||||
|
|
||||||
|
Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
|
||||||
|
|
||||||
|
for (ManualEntity manualEntity : manualEntities) {
|
||||||
|
|
||||||
|
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD);
|
||||||
|
|
||||||
|
if (optionalTextEntity.isEmpty()) {
|
||||||
|
unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange());
|
||||||
|
resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst());
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove all temp entities from the graph
|
||||||
|
tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional<ManualResizeRedaction> optionalManualResizeRedaction) {
|
||||||
|
|
||||||
|
if (optionalManualResizeRedaction.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
|
||||||
|
|
||||||
|
PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
|
||||||
|
.stream()
|
||||||
|
.filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
|
||||||
|
.findFirst()
|
||||||
|
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
|
||||||
|
|
||||||
|
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
|
||||||
|
|
||||||
|
String value = manualResizeRedaction.getValue();
|
||||||
|
int newStartOffset = -1;
|
||||||
|
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
|
||||||
|
ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
|
||||||
|
|
||||||
|
// Loop through nodes starting from the deepest fully containing node all the way to the document node
|
||||||
|
while (node != null) {
|
||||||
|
if (node.containsString(value)) {
|
||||||
|
SearchImplementation searchImplementation = new SearchImplementation(value, false);
|
||||||
|
List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
|
||||||
|
|
||||||
|
for (TextRange textRange : textRanges) {
|
||||||
|
SemanticNode finalNode = node;
|
||||||
|
|
||||||
|
List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
|
||||||
|
.stream()
|
||||||
|
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
// If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
|
||||||
|
ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
|
||||||
|
|
||||||
|
// Remove all temp entities from the graph
|
||||||
|
tempEntities.forEach(TextEntity::removeFromGraph);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
|
||||||
|
if (node.hasParent()) {
|
||||||
|
node = node.getParent();
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (closestEntity.getTextRange() != null) {
|
||||||
|
newStartOffset = closestEntity.getTextRange().start();
|
||||||
|
}
|
||||||
|
|
||||||
|
// need to reinsert the entity, due to the boundary having changed.
|
||||||
|
removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
|
||||||
|
entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
|
||||||
|
|
||||||
|
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
|
||||||
|
entityToBeResized.removeFromGraph();
|
||||||
|
entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
|
||||||
|
entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
|
||||||
|
entityToBeResized.setIntersectingNodes(new LinkedList<>());
|
||||||
|
entityToBeResized.setDeepestFullyContainingNode(null);
|
||||||
|
entityToBeResized.setPages(new HashSet<>());
|
||||||
|
entityToBeResized.getTextRange().setStart(newStartOffset);
|
||||||
|
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||||
|
// Don't insert into the graph if newStartOffset is -1 because it means nothing was found.
|
||||||
|
if (newStartOffset > -1) {
|
||||||
|
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
|
||||||
|
|
||||||
|
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
|
||||||
|
|
||||||
|
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
|
||||||
|
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
|
||||||
|
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
|
||||||
|
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
|
||||||
|
|
||||||
|
List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
|
||||||
|
for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
|
||||||
|
redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
|
||||||
|
}
|
||||||
|
correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
|
||||||
|
|
||||||
|
return correctEntity;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) {
|
||||||
|
|
||||||
|
return UnprocessedManualEntity.builder()
|
||||||
.annotationId(manualEntity.getId())
|
.annotationId(manualEntity.getId())
|
||||||
.textAfter("")
|
.textAfter("")
|
||||||
.textBefore("")
|
.textBefore("")
|
||||||
@ -86,21 +260,7 @@ public class UnprocessedChangesService {
|
|||||||
.stream()
|
.stream()
|
||||||
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||||
.toList())
|
.toList())
|
||||||
.build()));
|
.build();
|
||||||
|
|
||||||
rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
|
|
||||||
AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
|
|
||||||
|
|
||||||
return manualRedactions.getEntriesToAdd()
|
|
||||||
.stream()
|
|
||||||
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
|
|
||||||
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
|
|
||||||
manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only")))
|
|
||||||
.toList();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
|||||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
|
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
||||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||||
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
|
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
|
||||||
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
|
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
|
||||||
@ -115,6 +116,9 @@ public abstract class AbstractRedactionIntegrationTest {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private LayoutParsingPipeline layoutParsingPipeline;
|
private LayoutParsingPipeline layoutParsingPipeline;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
protected UnprocessedChangesService unprocessedChangesService;
|
||||||
|
|
||||||
@MockBean
|
@MockBean
|
||||||
protected RabbitTemplate rabbitTemplate;
|
protected RabbitTemplate rabbitTemplate;
|
||||||
|
|
||||||
|
|||||||
@ -4,18 +4,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.mockito.ArgumentCaptor;
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.mockito.Captor;
|
import org.mockito.Captor;
|
||||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||||
import org.springframework.boot.test.context.SpringBootTest;
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
@ -30,28 +33,36 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
|||||||
|
|
||||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
|
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
|
||||||
import com.iqser.red.service.redaction.v1.model.QueueNames;
|
import com.iqser.red.service.redaction.v1.model.QueueNames;
|
||||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
|
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
|
||||||
import com.iqser.red.service.redaction.v1.server.Application;
|
import com.iqser.red.service.redaction.v1.server.Application;
|
||||||
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
|
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
|
||||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||||
import com.iqser.red.storage.commons.service.StorageService;
|
import com.iqser.red.storage.commons.service.StorageService;
|
||||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
|
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
|
||||||
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
|
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
|
||||||
|
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||||
|
|
||||||
|
import lombok.SneakyThrows;
|
||||||
|
|
||||||
@ExtendWith(SpringExtension.class)
|
@ExtendWith(SpringExtension.class)
|
||||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||||
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
|
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
|
||||||
public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
|
public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
|
||||||
|
|
||||||
|
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||||
@Import({LayoutParsingServiceProcessorConfiguration.class})
|
@Import({LayoutParsingServiceProcessorConfiguration.class})
|
||||||
@ -73,63 +84,60 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
@SpyBean
|
@SpyBean
|
||||||
RabbitTemplate rabbitTemplate;
|
RabbitTemplate rabbitTemplate;
|
||||||
|
|
||||||
@Autowired
|
@BeforeEach
|
||||||
UnprocessedChangesService unprocessedChangesService;
|
public void stubClients() {
|
||||||
|
|
||||||
|
TenantContext.setTenantId("redaction");
|
||||||
|
|
||||||
|
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
|
||||||
|
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
|
||||||
|
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
|
||||||
|
|
||||||
|
loadDictionaryForTest();
|
||||||
|
loadTypeForTest();
|
||||||
|
loadNerForTest();
|
||||||
|
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||||
|
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
|
||||||
|
|
||||||
|
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||||
|
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
|
||||||
|
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||||
|
.type(DOSSIER_REDACTIONS_INDICATOR)
|
||||||
|
.dossierTemplateId(TEST_DOSSIER_ID)
|
||||||
|
.hexColor("#ffe187")
|
||||||
|
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||||
|
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||||
|
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||||
|
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||||
|
.build()));
|
||||||
|
|
||||||
|
mockDictionaryCalls(null);
|
||||||
|
|
||||||
|
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SneakyThrows
|
||||||
public void testManualSurroundingText() {
|
public void testManualSurroundingText() {
|
||||||
|
|
||||||
String pdfFile = "files/new/S4.pdf";
|
String pdfFile = "files/new/S4.pdf";
|
||||||
|
|
||||||
ManualRedactions manualRedactions = new ManualRedactions();
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
|
|
||||||
manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString());
|
|
||||||
manualRedactionEntry.setFileId("fileId");
|
|
||||||
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
|
|
||||||
manualRedactionEntry.setType("CBI_author");
|
|
||||||
manualRedactionEntry.setValue("rabbits");
|
|
||||||
manualRedactionEntry.setReason("Manual Redaction");
|
|
||||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
|
|
||||||
|
|
||||||
ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
|
|
||||||
manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
|
|
||||||
manualRedactionEntry2.setFileId("fileId");
|
|
||||||
manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED);
|
|
||||||
manualRedactionEntry2.setType("CBI_author");
|
|
||||||
manualRedactionEntry2.setValue("rabbits");
|
|
||||||
manualRedactionEntry2.setReason("Manual Redaction");
|
|
||||||
manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
|
|
||||||
|
|
||||||
var aoelId = UUID.randomUUID().toString();
|
var aoelId = UUID.randomUUID().toString();
|
||||||
ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
|
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL");
|
||||||
manualRedactionEntry3.setAnnotationId(aoelId);
|
|
||||||
manualRedactionEntry3.setFileId("fileId");
|
|
||||||
manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED);
|
|
||||||
manualRedactionEntry3.setType("CBI_author");
|
|
||||||
manualRedactionEntry3.setValue("AOEL");
|
|
||||||
manualRedactionEntry3.setReason("Manual Redaction");
|
|
||||||
manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
|
|
||||||
|
|
||||||
var notFoundId = UUID.randomUUID().toString();
|
var notFoundId = UUID.randomUUID().toString();
|
||||||
ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry();
|
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random");
|
||||||
manualRedactionEntry4.setAnnotationId(notFoundId);
|
|
||||||
manualRedactionEntry4.setFileId("fileId");
|
|
||||||
manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED);
|
|
||||||
manualRedactionEntry4.setType("CBI_author");
|
|
||||||
manualRedactionEntry4.setValue("Random");
|
|
||||||
manualRedactionEntry4.setReason("Manual Redaction");
|
|
||||||
manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()));
|
|
||||||
|
|
||||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
||||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
|
|
||||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry4);
|
|
||||||
|
|
||||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||||
request.setManualRedactions(manualRedactions);
|
|
||||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
request.setManualRedactions(manualRedactions);
|
||||||
unprocessedChangesService.analyseSurroundingText(request);
|
unprocessedChangesService.analyseSurroundingText(request);
|
||||||
|
|
||||||
verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||||
@ -137,7 +145,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||||
|
|
||||||
assertFalse(unprocessedManualEntities.isEmpty());
|
assertFalse(unprocessedManualEntities.isEmpty());
|
||||||
assertEquals(unprocessedManualEntities.size(), 4);
|
assertEquals(unprocessedManualEntities.size(), 2);
|
||||||
|
|
||||||
Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
|
Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
|
||||||
assertTrue(optionalUnprocessedManualEntity.isPresent());
|
assertTrue(optionalUnprocessedManualEntity.isPresent());
|
||||||
@ -163,5 +171,232 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
|||||||
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
|
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
|
||||||
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
|
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
|
||||||
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
|
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
|
||||||
|
|
||||||
|
analyzeService.reanalyze(request);
|
||||||
|
|
||||||
|
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build());
|
||||||
|
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL");
|
||||||
|
|
||||||
|
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||||
|
unprocessedChangesService.analyseSurroundingText(request);
|
||||||
|
|
||||||
|
verify(rabbitTemplate, times(2)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||||
|
|
||||||
|
unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||||
|
|
||||||
|
assertFalse(unprocessedManualEntities.isEmpty());
|
||||||
|
assertEquals(unprocessedManualEntities.size(), 1);
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMultipleManualResizeRequests() {
|
||||||
|
|
||||||
|
String pdfFile = "files/new/S4.pdf";
|
||||||
|
|
||||||
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
|
var aoelId = UUID.randomUUID().toString();
|
||||||
|
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||||
|
|
||||||
|
var cormsId = UUID.randomUUID().toString();
|
||||||
|
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS");
|
||||||
|
|
||||||
|
var a9Id = UUID.randomUUID().toString();
|
||||||
|
ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G");
|
||||||
|
|
||||||
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
|
||||||
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
||||||
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||||
|
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||||
|
request.setManualRedactions(manualRedactions);
|
||||||
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
|
||||||
|
List<Rectangle> positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build());
|
||||||
|
List<Rectangle> positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build());
|
||||||
|
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
|
||||||
|
ManualResizeRedaction manualResizeRedaction2 = prepareManualSizeRedaction(cormsId, positions2, "CoRMS proposed");
|
||||||
|
ManualResizeRedaction manualResizeRedaction3 = prepareManualSizeRedaction(a9Id, positions3, "metolachlor in A9396G");
|
||||||
|
|
||||||
|
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction, manualResizeRedaction2, manualResizeRedaction3)).build());
|
||||||
|
unprocessedChangesService.analyseSurroundingText(request);
|
||||||
|
|
||||||
|
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||||
|
|
||||||
|
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||||
|
|
||||||
|
assertFalse(unprocessedManualEntities.isEmpty());
|
||||||
|
assertEquals(unprocessedManualEntities.size(), 3);
|
||||||
|
|
||||||
|
var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst();
|
||||||
|
assertTrue(resizedAoel.isPresent());
|
||||||
|
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
|
||||||
|
assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
|
||||||
|
assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||||
|
assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||||
|
assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||||
|
assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth());
|
||||||
|
assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight());
|
||||||
|
|
||||||
|
var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst();
|
||||||
|
assertTrue(cormsResized.isPresent());
|
||||||
|
assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
|
||||||
|
assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
|
||||||
|
assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
|
||||||
|
assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX());
|
||||||
|
assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY());
|
||||||
|
assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth());
|
||||||
|
assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight());
|
||||||
|
|
||||||
|
var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst();
|
||||||
|
assertTrue(a9Resized.isPresent());
|
||||||
|
assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
|
||||||
|
assertEquals(a9Resized.get().getTextBefore(), "data for S");
|
||||||
|
assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||||
|
assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX());
|
||||||
|
assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY());
|
||||||
|
assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth());
|
||||||
|
assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testManualResizeIfMultipleValuesInTheSameNode() {
|
||||||
|
|
||||||
|
String pdfFile = "files/new/S4.pdf";
|
||||||
|
|
||||||
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
var aoelId = UUID.randomUUID().toString();
|
||||||
|
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||||
|
|
||||||
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||||
|
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||||
|
request.setManualRedactions(manualRedactions);
|
||||||
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
|
||||||
|
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
|
||||||
|
|
||||||
|
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||||
|
unprocessedChangesService.analyseSurroundingText(request);
|
||||||
|
|
||||||
|
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||||
|
|
||||||
|
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||||
|
|
||||||
|
assertFalse(unprocessedManualEntities.isEmpty());
|
||||||
|
assertEquals(unprocessedManualEntities.size(), 1);
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testManualResizeInADifferentSection() {
|
||||||
|
|
||||||
|
String pdfFile = "files/new/S4.pdf";
|
||||||
|
|
||||||
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
var aoelId = UUID.randomUUID().toString();
|
||||||
|
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||||
|
|
||||||
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||||
|
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||||
|
request.setManualRedactions(manualRedactions);
|
||||||
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build());
|
||||||
|
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL");
|
||||||
|
|
||||||
|
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||||
|
unprocessedChangesService.analyseSurroundingText(request);
|
||||||
|
|
||||||
|
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||||
|
|
||||||
|
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||||
|
|
||||||
|
assertFalse(unprocessedManualEntities.isEmpty());
|
||||||
|
assertEquals(unprocessedManualEntities.size(), 1);
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
|
||||||
|
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testManualResizeNotFound() {
|
||||||
|
|
||||||
|
String pdfFile = "files/new/S4.pdf";
|
||||||
|
|
||||||
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
var aoelId = UUID.randomUUID().toString();
|
||||||
|
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||||
|
|
||||||
|
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||||
|
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||||
|
request.setManualRedactions(manualRedactions);
|
||||||
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
|
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
|
||||||
|
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist");
|
||||||
|
|
||||||
|
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||||
|
unprocessedChangesService.analyseSurroundingText(request);
|
||||||
|
|
||||||
|
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||||
|
|
||||||
|
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||||
|
|
||||||
|
assertTrue(unprocessedManualEntities.isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static ManualResizeRedaction prepareManualSizeRedaction(String id, List<Rectangle> positions, String value) {
|
||||||
|
|
||||||
|
ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction();
|
||||||
|
manualResizeRedaction.setAnnotationId(id);
|
||||||
|
manualResizeRedaction.setPositions(positions);
|
||||||
|
manualResizeRedaction.setUpdateDictionary(false);
|
||||||
|
manualResizeRedaction.setAddToAllDossiers(false);
|
||||||
|
manualResizeRedaction.setValue(value);
|
||||||
|
return manualResizeRedaction;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static ManualRedactionEntry prepareManualRedactionEntry(String id, List<Rectangle> positions, String value) {
|
||||||
|
|
||||||
|
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
|
||||||
|
manualRedactionEntry.setAnnotationId(id);
|
||||||
|
manualRedactionEntry.setFileId("fileId");
|
||||||
|
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
|
||||||
|
manualRedactionEntry.setType("CBI_author");
|
||||||
|
manualRedactionEntry.setValue(value);
|
||||||
|
manualRedactionEntry.setReason("Manual Redaction");
|
||||||
|
manualRedactionEntry.setPositions(positions);
|
||||||
|
return manualRedactionEntry;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -26,6 +26,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
|
|||||||
|
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
|
|
||||||
|
@SuppressWarnings("PMD")
|
||||||
class RuleFileFactoryTest {
|
class RuleFileFactoryTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@ -28,6 +28,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
|
|||||||
|
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
|
|
||||||
|
@SuppressWarnings("PMD")
|
||||||
class OldRulesParserTest {
|
class OldRulesParserTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user