RED-8043 - Calculate surrounding text for unprocessed manual resize
This commit is contained in:
Andrei Isvoran
parent
7b5c0026f9
commit
b42fae4054
@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Builder
|
||||
@Getter
|
||||
@Setter
|
||||
public class ClosestEntity {
|
||||
|
||||
private double distance;
|
||||
private TextRange textRange;
|
||||
|
||||
}
|
||||
@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.model;
|
||||
import java.util.List;
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
@ -66,6 +68,26 @@ public class ManualEntity implements IEntity {
|
||||
.build();
|
||||
}
|
||||
|
||||
public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
|
||||
|
||||
List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
|
||||
EntityType entityType = getEntityType(entityLogEntry.getEntryType());
|
||||
return ManualEntity.builder()
|
||||
.id(entityLogEntry.getId())
|
||||
.value(entityLogEntry.getValue())
|
||||
.entityPosition(rectangleWithPages)
|
||||
.ruleIdentifier(entityLogEntry.getMatchedRule())
|
||||
.reason(entityLogEntry.getReason())
|
||||
.legalBasis(entityLogEntry.getLegalBasis())
|
||||
.type(entityLogEntry.getType())
|
||||
.section(entityLogEntry.getSection())
|
||||
.entityType(entityType)
|
||||
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
|
||||
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
|
||||
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public TextRange getTextRange() {
|
||||
@ -80,6 +102,27 @@ public class ManualEntity implements IEntity {
|
||||
return getManualOverwrite().getType().orElse(type);
|
||||
}
|
||||
|
||||
private static EntityType getEntityType(EntryType entryType) {
|
||||
|
||||
switch (entryType) {
|
||||
case FALSE_RECOMMENDATION -> {
|
||||
return EntityType.FALSE_RECOMMENDATION;
|
||||
}
|
||||
case FALSE_POSITIVE -> {
|
||||
return EntityType.FALSE_POSITIVE;
|
||||
}
|
||||
case HINT -> {
|
||||
return EntityType.HINT;
|
||||
}
|
||||
case RECOMMENDATION -> {
|
||||
return EntityType.RECOMMENDATION;
|
||||
}
|
||||
default -> {
|
||||
return EntityType.ENTITY;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private EntityType getEntityType(boolean isHint) {
|
||||
return isHint ? EntityType.HINT : EntityType.ENTITY;
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
|
||||
public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
||||
@ -12,6 +13,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
||||
}
|
||||
|
||||
|
||||
public static RectangleWithPage fromEntityLogPosition(Position position) {
|
||||
|
||||
return new RectangleWithPage(position.getPageNumber(), toRectangle2D(position));
|
||||
}
|
||||
|
||||
|
||||
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||
|
||||
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
||||
@ -24,6 +31,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(Position position) {
|
||||
|
||||
return new Rectangle2D.Float(position.x(), position.y(), position.w(), position.h());
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||
|
||||
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
|
||||
|
||||
@ -3,10 +3,17 @@ package com.iqser.red.service.redaction.v1.server.service;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
@ -50,13 +57,46 @@ public class ManualChangesApplicationService {
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
|
||||
|
||||
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList());
|
||||
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
|
||||
|
||||
int newStartOffset;
|
||||
if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
|
||||
newStartOffset = entityToBeResized.getTextRange().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
|
||||
} else {
|
||||
newStartOffset = entityToBeResized.getTextRange().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
|
||||
String value = manualResizeRedaction.getValue();
|
||||
int newStartOffset = -1;
|
||||
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
|
||||
ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
|
||||
|
||||
// Loop through nodes starting from the deepest fully containing node all the way to the document node
|
||||
while (node != null) {
|
||||
if (node.containsString(value)) {
|
||||
SearchImplementation searchImplementation = new SearchImplementation(value, false);
|
||||
List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
|
||||
|
||||
for (TextRange textRange : textRanges) {
|
||||
SemanticNode finalNode = node;
|
||||
|
||||
List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
|
||||
.stream()
|
||||
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
|
||||
determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
|
||||
|
||||
// Remove all temp entities from the graph
|
||||
tempEntities.forEach(TextEntity::removeFromGraph);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
|
||||
if (node.hasParent()) {
|
||||
node = node.getParent();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (closestEntity.getTextRange() != null) {
|
||||
newStartOffset = closestEntity.getTextRange().start();
|
||||
}
|
||||
|
||||
// need to reinsert the entity, due to the boundary having changed.
|
||||
@ -65,6 +105,31 @@ public class ManualChangesApplicationService {
|
||||
}
|
||||
|
||||
|
||||
public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction,
|
||||
TextRange textRange,
|
||||
List<TextEntity> tempEntities,
|
||||
ClosestEntity closestEntity) {
|
||||
|
||||
double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage());
|
||||
if (currentDistance < closestEntity.getDistance()) {
|
||||
closestEntity.setDistance(currentDistance);
|
||||
closestEntity.setTextRange(textRange);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static double calculateClosest(Rectangle position, List<PositionOnPage> positionOnPages) {
|
||||
|
||||
Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0);
|
||||
double difference = 0;
|
||||
difference += position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX());
|
||||
difference += position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY());
|
||||
difference += position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth());
|
||||
difference += position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight());
|
||||
return difference;
|
||||
}
|
||||
|
||||
|
||||
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
|
||||
|
||||
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
|
||||
@ -75,7 +140,9 @@ public class ManualChangesApplicationService {
|
||||
entityToBeResized.setPages(new HashSet<>());
|
||||
entityToBeResized.getTextRange().setStart(newStartOffset);
|
||||
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||
if (newStartOffset > -1) {
|
||||
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -90,7 +157,7 @@ public class ManualChangesApplicationService {
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
|
||||
public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
|
||||
|
||||
return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
|
||||
}
|
||||
|
||||
@ -1,8 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@ -10,20 +15,32 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.QueueNames;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import io.micrometer.observation.annotation.Observed;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
@ -32,28 +49,52 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class UnprocessedChangesService {
|
||||
|
||||
ManualEntityCreationService manualEntityCreationService;
|
||||
RabbitTemplate rabbitTemplate;
|
||||
ObservedStorageService observedStorageService;
|
||||
private static final double THRESHOLD = 10;
|
||||
|
||||
final RabbitTemplate rabbitTemplate;
|
||||
final ObservedStorageService observedStorageService;
|
||||
final ManualRedactionEntryService manualRedactionEntryService;
|
||||
final EntityFindingUtility entityFindingUtility;
|
||||
final RedactionStorageService redactionStorageService;
|
||||
final EntityEnrichmentService entityEnrichmentService;
|
||||
EntityCreationService entityCreationService;
|
||||
|
||||
|
||||
@PostConstruct
|
||||
public void initEntityCreationService() {
|
||||
|
||||
entityCreationService = new EntityCreationService(entityEnrichmentService);
|
||||
}
|
||||
|
||||
|
||||
@Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
|
||||
public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
|
||||
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
Set<String> annotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
|
||||
annotationIds.addAll(analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()));
|
||||
|
||||
List<ManualEntity> notFoundManualEntities = new ArrayList<>();
|
||||
List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions());
|
||||
if (!manualEntities.isEmpty()) {
|
||||
notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
|
||||
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
|
||||
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
|
||||
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
|
||||
allAnnotationIds.addAll(resizeIds);
|
||||
|
||||
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
|
||||
List<ManualEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
|
||||
.stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList()
|
||||
.stream().map(ManualEntity::fromEntityLogEntry).toList();
|
||||
|
||||
if (!manualResizeRedactions.isEmpty()) {
|
||||
processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
|
||||
}
|
||||
|
||||
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
|
||||
document,
|
||||
analyzeRequest.getDossierTemplateId());
|
||||
|
||||
document.getEntities().forEach(textEntity -> {
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
|
||||
@ -64,9 +105,9 @@ public class UnprocessedChangesService {
|
||||
List<Position> positions = positionsOnPerPage.getRectanglePerLine()
|
||||
.stream()
|
||||
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
|
||||
.toList();
|
||||
.collect(Collectors.toList());
|
||||
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
||||
.annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
|
||||
.annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
|
||||
.textBefore(textEntity.getTextBefore())
|
||||
.textAfter(textEntity.getTextAfter())
|
||||
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
|
||||
@ -75,7 +116,140 @@ public class UnprocessedChangesService {
|
||||
}
|
||||
});
|
||||
|
||||
notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
||||
notFoundManualRedactionEntries.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
|
||||
|
||||
rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
|
||||
AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
|
||||
}
|
||||
|
||||
|
||||
private void processManualResizeRedactions(Document document,
|
||||
List<ManualEntity> manualEntities,
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities,
|
||||
List<ManualResizeRedaction> manualResizeRedactions) {
|
||||
|
||||
Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
|
||||
|
||||
for (ManualEntity manualEntity : manualEntities) {
|
||||
|
||||
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD);
|
||||
|
||||
if (optionalTextEntity.isEmpty()) {
|
||||
unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity));
|
||||
continue;
|
||||
}
|
||||
|
||||
TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange());
|
||||
resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst());
|
||||
}
|
||||
|
||||
// remove all temp entities from the graph
|
||||
tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
|
||||
}
|
||||
|
||||
|
||||
public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional<ManualResizeRedaction> optionalManualResizeRedaction) {
|
||||
|
||||
if (optionalManualResizeRedaction.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
|
||||
|
||||
PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
|
||||
.stream()
|
||||
.filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
|
||||
|
||||
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
|
||||
|
||||
String value = manualResizeRedaction.getValue();
|
||||
int newStartOffset = -1;
|
||||
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
|
||||
ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
|
||||
|
||||
// Loop through nodes starting from the deepest fully containing node all the way to the document node
|
||||
while (node != null) {
|
||||
if (node.containsString(value)) {
|
||||
SearchImplementation searchImplementation = new SearchImplementation(value, false);
|
||||
List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
|
||||
|
||||
for (TextRange textRange : textRanges) {
|
||||
SemanticNode finalNode = node;
|
||||
|
||||
List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
|
||||
.stream()
|
||||
.map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
|
||||
ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
|
||||
|
||||
// Remove all temp entities from the graph
|
||||
tempEntities.forEach(TextEntity::removeFromGraph);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
|
||||
if (node.hasParent()) {
|
||||
node = node.getParent();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (closestEntity.getTextRange() != null) {
|
||||
newStartOffset = closestEntity.getTextRange().start();
|
||||
}
|
||||
|
||||
// need to reinsert the entity, due to the boundary having changed.
|
||||
removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
|
||||
entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
|
||||
}
|
||||
|
||||
|
||||
private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
|
||||
|
||||
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
|
||||
entityToBeResized.removeFromGraph();
|
||||
entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
|
||||
entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
|
||||
entityToBeResized.setIntersectingNodes(new LinkedList<>());
|
||||
entityToBeResized.setDeepestFullyContainingNode(null);
|
||||
entityToBeResized.setPages(new HashSet<>());
|
||||
entityToBeResized.getTextRange().setStart(newStartOffset);
|
||||
entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||
// Don't insert into the graph if newStartOffset is -1 because it means nothing was found.
|
||||
if (newStartOffset > -1) {
|
||||
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
|
||||
|
||||
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
|
||||
|
||||
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
|
||||
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
|
||||
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
|
||||
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
|
||||
|
||||
List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
|
||||
for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
|
||||
redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
|
||||
}
|
||||
correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
|
||||
|
||||
return correctEntity;
|
||||
}
|
||||
|
||||
|
||||
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) {
|
||||
|
||||
return UnprocessedManualEntity.builder()
|
||||
.annotationId(manualEntity.getId())
|
||||
.textAfter("")
|
||||
.textBefore("")
|
||||
@ -86,21 +260,7 @@ public class UnprocessedChangesService {
|
||||
.stream()
|
||||
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||
.toList())
|
||||
.build()));
|
||||
|
||||
rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
|
||||
AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
|
||||
}
|
||||
|
||||
|
||||
private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
|
||||
|
||||
return manualRedactions.getEntriesToAdd()
|
||||
.stream()
|
||||
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
|
||||
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
|
||||
manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only")))
|
||||
.toList();
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
|
||||
@ -115,6 +116,9 @@ public abstract class AbstractRedactionIntegrationTest {
|
||||
@Autowired
|
||||
private LayoutParsingPipeline layoutParsingPipeline;
|
||||
|
||||
@Autowired
|
||||
protected UnprocessedChangesService unprocessedChangesService;
|
||||
|
||||
@MockBean
|
||||
protected RabbitTemplate rabbitTemplate;
|
||||
|
||||
|
||||
@ -4,18 +4,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Captor;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
@ -30,28 +33,36 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.QueueNames;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
|
||||
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
|
||||
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
|
||||
public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
|
||||
|
||||
private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@Import({LayoutParsingServiceProcessorConfiguration.class})
|
||||
@ -73,63 +84,60 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
||||
@SpyBean
|
||||
RabbitTemplate rabbitTemplate;
|
||||
|
||||
@Autowired
|
||||
UnprocessedChangesService unprocessedChangesService;
|
||||
@BeforeEach
|
||||
public void stubClients() {
|
||||
|
||||
TenantContext.setTenantId("redaction");
|
||||
|
||||
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
|
||||
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
|
||||
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
|
||||
|
||||
loadDictionaryForTest();
|
||||
loadTypeForTest();
|
||||
loadNerForTest();
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
|
||||
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.type(DOSSIER_REDACTIONS_INDICATOR)
|
||||
.dossierTemplateId(TEST_DOSSIER_ID)
|
||||
.hexColor("#ffe187")
|
||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
|
||||
.build()));
|
||||
|
||||
mockDictionaryCalls(null);
|
||||
|
||||
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testManualSurroundingText() {
|
||||
|
||||
String pdfFile = "files/new/S4.pdf";
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
|
||||
manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString());
|
||||
manualRedactionEntry.setFileId("fileId");
|
||||
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry.setType("CBI_author");
|
||||
manualRedactionEntry.setValue("rabbits");
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
|
||||
manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
|
||||
manualRedactionEntry2.setFileId("fileId");
|
||||
manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry2.setType("CBI_author");
|
||||
manualRedactionEntry2.setValue("rabbits");
|
||||
manualRedactionEntry2.setReason("Manual Redaction");
|
||||
manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
|
||||
|
||||
var aoelId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
|
||||
manualRedactionEntry3.setAnnotationId(aoelId);
|
||||
manualRedactionEntry3.setFileId("fileId");
|
||||
manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry3.setType("CBI_author");
|
||||
manualRedactionEntry3.setValue("AOEL");
|
||||
manualRedactionEntry3.setReason("Manual Redaction");
|
||||
manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
|
||||
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL");
|
||||
|
||||
var notFoundId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry();
|
||||
manualRedactionEntry4.setAnnotationId(notFoundId);
|
||||
manualRedactionEntry4.setFileId("fileId");
|
||||
manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry4.setType("CBI_author");
|
||||
manualRedactionEntry4.setValue("Random");
|
||||
manualRedactionEntry4.setReason("Manual Redaction");
|
||||
manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()));
|
||||
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random");
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry4);
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
request.setManualRedactions(manualRedactions);
|
||||
unprocessedChangesService.analyseSurroundingText(request);
|
||||
|
||||
verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||
@ -137,7 +145,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||
|
||||
assertFalse(unprocessedManualEntities.isEmpty());
|
||||
assertEquals(unprocessedManualEntities.size(), 4);
|
||||
assertEquals(unprocessedManualEntities.size(), 2);
|
||||
|
||||
Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
|
||||
assertTrue(optionalUnprocessedManualEntity.isPresent());
|
||||
@ -163,5 +171,232 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
|
||||
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
|
||||
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
|
||||
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build());
|
||||
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL");
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||
unprocessedChangesService.analyseSurroundingText(request);
|
||||
|
||||
verify(rabbitTemplate, times(2)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||
|
||||
unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||
|
||||
assertFalse(unprocessedManualEntities.isEmpty());
|
||||
assertEquals(unprocessedManualEntities.size(), 1);
|
||||
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
|
||||
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
|
||||
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
|
||||
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testMultipleManualResizeRequests() {
|
||||
|
||||
String pdfFile = "files/new/S4.pdf";
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
|
||||
var aoelId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||
|
||||
var cormsId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS");
|
||||
|
||||
var a9Id = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G");
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
|
||||
List<Rectangle> positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build());
|
||||
List<Rectangle> positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build());
|
||||
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
|
||||
ManualResizeRedaction manualResizeRedaction2 = prepareManualSizeRedaction(cormsId, positions2, "CoRMS proposed");
|
||||
ManualResizeRedaction manualResizeRedaction3 = prepareManualSizeRedaction(a9Id, positions3, "metolachlor in A9396G");
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction, manualResizeRedaction2, manualResizeRedaction3)).build());
|
||||
unprocessedChangesService.analyseSurroundingText(request);
|
||||
|
||||
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||
|
||||
assertFalse(unprocessedManualEntities.isEmpty());
|
||||
assertEquals(unprocessedManualEntities.size(), 3);
|
||||
|
||||
var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst();
|
||||
assertTrue(resizedAoel.isPresent());
|
||||
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
|
||||
assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
|
||||
assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||
assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||
assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||
assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth());
|
||||
assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight());
|
||||
|
||||
var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst();
|
||||
assertTrue(cormsResized.isPresent());
|
||||
assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
|
||||
assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
|
||||
assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
|
||||
assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX());
|
||||
assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY());
|
||||
assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth());
|
||||
assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight());
|
||||
|
||||
var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst();
|
||||
assertTrue(a9Resized.isPresent());
|
||||
assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
|
||||
assertEquals(a9Resized.get().getTextBefore(), "data for S");
|
||||
assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||
assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX());
|
||||
assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY());
|
||||
assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth());
|
||||
assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testManualResizeIfMultipleValuesInTheSameNode() {
|
||||
|
||||
String pdfFile = "files/new/S4.pdf";
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
var aoelId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
|
||||
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||
unprocessedChangesService.analyseSurroundingText(request);
|
||||
|
||||
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||
|
||||
assertFalse(unprocessedManualEntities.isEmpty());
|
||||
assertEquals(unprocessedManualEntities.size(), 1);
|
||||
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
|
||||
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
|
||||
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
|
||||
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testManualResizeInADifferentSection() {
|
||||
|
||||
String pdfFile = "files/new/S4.pdf";
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
var aoelId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build());
|
||||
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL");
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||
unprocessedChangesService.analyseSurroundingText(request);
|
||||
|
||||
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||
|
||||
assertFalse(unprocessedManualEntities.isEmpty());
|
||||
assertEquals(unprocessedManualEntities.size(), 1);
|
||||
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
|
||||
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
|
||||
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
|
||||
assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
|
||||
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testManualResizeNotFound() {
|
||||
|
||||
String pdfFile = "files/new/S4.pdf";
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
var aoelId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
analyzeService.analyze(request);
|
||||
|
||||
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
|
||||
ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist");
|
||||
|
||||
request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
|
||||
unprocessedChangesService.analyseSurroundingText(request);
|
||||
|
||||
verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
|
||||
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
|
||||
|
||||
assertTrue(unprocessedManualEntities.isEmpty());
|
||||
}
|
||||
|
||||
|
||||
private static ManualResizeRedaction prepareManualSizeRedaction(String id, List<Rectangle> positions, String value) {
|
||||
|
||||
ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction();
|
||||
manualResizeRedaction.setAnnotationId(id);
|
||||
manualResizeRedaction.setPositions(positions);
|
||||
manualResizeRedaction.setUpdateDictionary(false);
|
||||
manualResizeRedaction.setAddToAllDossiers(false);
|
||||
manualResizeRedaction.setValue(value);
|
||||
return manualResizeRedaction;
|
||||
}
|
||||
|
||||
|
||||
private static ManualRedactionEntry prepareManualRedactionEntry(String id, List<Rectangle> positions, String value) {
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
|
||||
manualRedactionEntry.setAnnotationId(id);
|
||||
manualRedactionEntry.setFileId("fileId");
|
||||
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry.setType("CBI_author");
|
||||
manualRedactionEntry.setValue(value);
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setPositions(positions);
|
||||
return manualRedactionEntry;
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,6 +26,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@SuppressWarnings("PMD")
|
||||
class RuleFileFactoryTest {
|
||||
|
||||
@Test
|
||||
|
||||
@ -28,6 +28,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@SuppressWarnings("PMD")
|
||||
class OldRulesParserTest {
|
||||
|
||||
@Test
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user