From b42fae4054eaa4d90ce97c8d5136e49033751681 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Mon, 8 Jan 2024 14:15:38 +0100 Subject: [PATCH] RED-8043 - Calculate surrounding text for unprocessed manual resize --- .../v1/server/model/ClosestEntity.java | 17 + .../v1/server/model/ManualEntity.java | 43 +++ .../v1/server/model/RectangleWithPage.java | 13 + .../ManualChangesApplicationService.java | 83 ++++- .../service/UnprocessedChangesService.java | 222 ++++++++++-- .../AbstractRedactionIntegrationTest.java | 4 + .../UnprocessedChangesServiceTest.java | 319 +++++++++++++++--- .../factory/RuleFileFactoryTest.java | 1 + .../translation/OldRulesParserTest.java | 1 + 9 files changed, 622 insertions(+), 81 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java new file mode 100644 index 00000000..275f7e83 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +@Builder +@Getter +@Setter +public class ClosestEntity { + + private double distance; + private TextRange textRange; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java index c06d089a..daa5df20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java @@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.model; import java.util.List; import java.util.PriorityQueue; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -66,6 +68,26 @@ public class ManualEntity implements IEntity { .build(); } + public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) { + + List rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList(); + EntityType entityType = getEntityType(entityLogEntry.getEntryType()); + return ManualEntity.builder() + .id(entityLogEntry.getId()) + .value(entityLogEntry.getValue()) + .entityPosition(rectangleWithPages) + .ruleIdentifier(entityLogEntry.getMatchedRule()) + .reason(entityLogEntry.getReason()) + .legalBasis(entityLogEntry.getLegalBasis()) + .type(entityLogEntry.getType()) + .section(entityLogEntry.getSection()) + .entityType(entityType) + .isDictionaryEntry(entityLogEntry.isDictionaryEntry()) + .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry()) + .manualOverwrite(new ManualChangeOverwrite(entityType)) + .build(); + } + @Override public TextRange getTextRange() { @@ -80,6 +102,27 @@ public class ManualEntity implements IEntity { return getManualOverwrite().getType().orElse(type); } + private static EntityType getEntityType(EntryType entryType) { + + switch (entryType) { + case FALSE_RECOMMENDATION -> { + return EntityType.FALSE_RECOMMENDATION; + } + case FALSE_POSITIVE -> { + return EntityType.FALSE_POSITIVE; + } + case HINT -> { + return EntityType.HINT; + } + case RECOMMENDATION -> { + return EntityType.RECOMMENDATION; + } + default -> { + return EntityType.ENTITY; + } + } + } + private EntityType getEntityType(boolean isHint) { return isHint ? EntityType.HINT : EntityType.ENTITY; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java index a04f9658..447dd4d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model; import java.awt.geom.Rectangle2D; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { @@ -12,6 +13,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { } + public static RectangleWithPage fromEntityLogPosition(Position position) { + + return new RectangleWithPage(position.getPageNumber(), toRectangle2D(position)); + } + + public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) { return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle)); @@ -24,6 +31,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { } + private static Rectangle2D toRectangle2D(Position position) { + + return new Rectangle2D.Float(position.x(), position.y(), position.w(), position.h()); + } + + private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) { return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 0ad87d1a..c6384869 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -3,10 +3,17 @@ package com.iqser.red.service.redaction.v1.server.service; import java.awt.geom.Rectangle2D; import java.util.HashSet; import java.util.LinkedList; +import java.util.List; import java.util.NoSuchElementException; +import java.util.stream.Collectors; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -50,13 +57,46 @@ public class ManualChangesApplicationService { .findFirst() .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); - positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList()); + positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList())); - int newStartOffset; - if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) { - newStartOffset = entityToBeResized.getTextRange().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue()); - } else { - newStartOffset = entityToBeResized.getTextRange().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue()); + String value = manualResizeRedaction.getValue(); + int newStartOffset = -1; + SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); + ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build(); + + // Loop through nodes starting from the deepest fully containing node all the way to the document node + while (node != null) { + if (node.containsString(value)) { + SearchImplementation searchImplementation = new SearchImplementation(value, false); + List textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()); + + for (TextRange textRange : textRanges) { + SemanticNode finalNode = node; + + List tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange) + .stream() + .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode)) + .collect(Collectors.toList()); + + // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one. + determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity); + + // Remove all temp entities from the graph + tempEntities.forEach(TextEntity::removeFromGraph); + } + break; + } + + // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere. + if (node.hasParent()) { + node = node.getParent(); + } else { + break; + } + } + + if (closestEntity.getTextRange() != null) { + newStartOffset = closestEntity.getTextRange().start(); } // need to reinsert the entity, due to the boundary having changed. @@ -65,6 +105,31 @@ public class ManualChangesApplicationService { } + public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction, + TextRange textRange, + List tempEntities, + ClosestEntity closestEntity) { + + double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage()); + if (currentDistance < closestEntity.getDistance()) { + closestEntity.setDistance(currentDistance); + closestEntity.setTextRange(textRange); + } + } + + + private static double calculateClosest(Rectangle position, List positionOnPages) { + + Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0); + double difference = 0; + difference += position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX()); + difference += position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY()); + difference += position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth()); + difference += position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight()); + return difference; + } + + private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) { SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode(); @@ -75,7 +140,9 @@ public class ManualChangesApplicationService { entityToBeResized.setPages(new HashSet<>()); entityToBeResized.getTextRange().setStart(newStartOffset); entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); - entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); + if (newStartOffset > -1) { + entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); + } } @@ -90,7 +157,7 @@ public class ManualChangesApplicationService { } - private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) { + public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) { return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java index 219f729e..125990cc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java @@ -1,8 +1,13 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -10,20 +15,32 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; +import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; +import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import io.micrometer.observation.annotation.Observed; +import jakarta.annotation.PostConstruct; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @@ -32,28 +49,52 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@FieldDefaults(level = AccessLevel.PRIVATE) public class UnprocessedChangesService { - ManualEntityCreationService manualEntityCreationService; - RabbitTemplate rabbitTemplate; - ObservedStorageService observedStorageService; + private static final double THRESHOLD = 10; + + final RabbitTemplate rabbitTemplate; + final ObservedStorageService observedStorageService; + final ManualRedactionEntryService manualRedactionEntryService; + final EntityFindingUtility entityFindingUtility; + final RedactionStorageService redactionStorageService; + final EntityEnrichmentService entityEnrichmentService; + EntityCreationService entityCreationService; + + + @PostConstruct + public void initEntityCreationService() { + + entityCreationService = new EntityCreationService(entityEnrichmentService); + } @Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text") public void analyseSurroundingText(AnalyzeRequest analyzeRequest) { List unprocessedManualEntities = new ArrayList<>(); - Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - Set annotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet()); - annotationIds.addAll(analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet())); - List notFoundManualEntities = new ArrayList<>(); - List manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions()); - if (!manualEntities.isEmpty()) { - notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document); + EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + + Set allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet()); + Set resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()); + allAnnotationIds.addAll(resizeIds); + + List manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList(); + List manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry() + .stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList() + .stream().map(ManualEntity::fromEntityLogEntry).toList(); + + if (!manualResizeRedactions.isEmpty()) { + processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions); } + List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, + document, + analyzeRequest.getDossierTemplateId()); + document.getEntities().forEach(textEntity -> { Set processedIds = new HashSet<>(); for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) { @@ -64,9 +105,9 @@ public class UnprocessedChangesService { List positions = positionsOnPerPage.getRectanglePerLine() .stream() .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber())) - .toList(); + .collect(Collectors.toList()); unprocessedManualEntities.add(UnprocessedManualEntity.builder() - .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse("")) + .annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse("")) .textBefore(textEntity.getTextBefore()) .textAfter(textEntity.getTextAfter()) .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString())) @@ -75,7 +116,140 @@ public class UnprocessedChangesService { } }); - notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(UnprocessedManualEntity.builder() + notFoundManualRedactionEntries.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity))); + + rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE, + AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build()); + } + + + private void processManualResizeRedactions(Document document, + List manualEntities, + List unprocessedManualEntities, + List manualResizeRedactions) { + + Map> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities); + + for (ManualEntity manualEntity : manualEntities) { + + Optional optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD); + + if (optionalTextEntity.isEmpty()) { + unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)); + continue; + } + + TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange()); + resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst()); + } + + // remove all temp entities from the graph + tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + } + + + public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional optionalManualResizeRedaction) { + + if (optionalManualResizeRedaction.isEmpty()) { + return; + } + + ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); + + PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage() + .stream() + .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId())) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); + + positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList())); + + String value = manualResizeRedaction.getValue(); + int newStartOffset = -1; + SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); + ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build(); + + // Loop through nodes starting from the deepest fully containing node all the way to the document node + while (node != null) { + if (node.containsString(value)) { + SearchImplementation searchImplementation = new SearchImplementation(value, false); + List textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()); + + for (TextRange textRange : textRanges) { + SemanticNode finalNode = node; + + List tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange) + .stream() + .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode)) + .collect(Collectors.toList()); + + // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one. + ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity); + + // Remove all temp entities from the graph + tempEntities.forEach(TextEntity::removeFromGraph); + } + break; + } + + // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere. + if (node.hasParent()) { + node = node.getParent(); + } else { + break; + } + } + + if (closestEntity.getTextRange() != null) { + newStartOffset = closestEntity.getTextRange().start(); + } + + // need to reinsert the entity, due to the boundary having changed. + removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset); + entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction); + } + + + private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) { + + SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode(); + entityToBeResized.removeFromGraph(); + entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this)); + entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this)); + entityToBeResized.setIntersectingNodes(new LinkedList<>()); + entityToBeResized.setDeepestFullyContainingNode(null); + entityToBeResized.setPages(new HashSet<>()); + entityToBeResized.getTextRange().setStart(newStartOffset); + entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); + // Don't insert into the graph if newStartOffset is -1 because it means nothing was found. + if (newStartOffset > -1) { + entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); + } + } + + + private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) { + + TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node); + + correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); + correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); + correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); + correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); + + List redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size()); + for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) { + redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine())); + } + correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage); + + return correctEntity; + } + + + private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) { + + return UnprocessedManualEntity.builder() .annotationId(manualEntity.getId()) .textAfter("") .textBefore("") @@ -86,21 +260,7 @@ public class UnprocessedChangesService { .stream() .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .toList()) - .build())); - - rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE, - AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build()); - } - - - private List manualEntitiesConverter(ManualRedactions manualRedactions) { - - return manualRedactions.getEntriesToAdd() - .stream() - .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty()) - .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, - manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only"))) - .toList(); + .build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 8df94792..69b40736 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; @@ -115,6 +116,9 @@ public abstract class AbstractRedactionIntegrationTest { @Autowired private LayoutParsingPipeline layoutParsingPipeline; + @Autowired + protected UnprocessedChangesService unprocessedChangesService; + @MockBean protected RabbitTemplate rabbitTemplate; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index c13b4620..7af446db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -4,18 +4,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; @@ -30,28 +33,36 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest; import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest; -import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest { + private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); + @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @Import({LayoutParsingServiceProcessorConfiguration.class}) @@ -73,63 +84,60 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT @SpyBean RabbitTemplate rabbitTemplate; - @Autowired - UnprocessedChangesService unprocessedChangesService; + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } @Test + @SneakyThrows public void testManualSurroundingText() { String pdfFile = "files/new/S4.pdf"; ManualRedactions manualRedactions = new ManualRedactions(); - ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); - manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString()); - manualRedactionEntry.setFileId("fileId"); - manualRedactionEntry.setStatus(AnnotationStatus.APPROVED); - manualRedactionEntry.setType("CBI_author"); - manualRedactionEntry.setValue("rabbits"); - manualRedactionEntry.setReason("Manual Redaction"); - manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build())); - - ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry(); - manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString()); - manualRedactionEntry2.setFileId("fileId"); - manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED); - manualRedactionEntry2.setType("CBI_author"); - manualRedactionEntry2.setValue("rabbits"); - manualRedactionEntry2.setReason("Manual Redaction"); - manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build())); - var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry(); - manualRedactionEntry3.setAnnotationId(aoelId); - manualRedactionEntry3.setFileId("fileId"); - manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED); - manualRedactionEntry3.setType("CBI_author"); - manualRedactionEntry3.setValue("AOEL"); - manualRedactionEntry3.setReason("Manual Redaction"); - manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build())); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL"); var notFoundId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry(); - manualRedactionEntry4.setAnnotationId(notFoundId); - manualRedactionEntry4.setFileId("fileId"); - manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED); - manualRedactionEntry4.setType("CBI_author"); - manualRedactionEntry4.setValue("Random"); - manualRedactionEntry4.setReason("Manual Redaction"); - manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build())); + ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); - manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); - manualRedactions.getEntriesToAdd().add(manualRedactionEntry4); AnalyzeRequest request = uploadFileToStorage(pdfFile); - request.setManualRedactions(manualRedactions); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + request.setManualRedactions(manualRedactions); unprocessedChangesService.analyseSurroundingText(request); verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); @@ -137,7 +145,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); assertFalse(unprocessedManualEntities.isEmpty()); - assertEquals(unprocessedManualEntities.size(), 4); + assertEquals(unprocessedManualEntities.size(), 2); Optional optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst(); assertTrue(optionalUnprocessedManualEntity.isPresent()); @@ -163,5 +171,232 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f); assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f); assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f); + + analyzeService.reanalyze(request); + + List positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build()); + ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL"); + + request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); + unprocessedChangesService.analyseSurroundingText(request); + + verify(rabbitTemplate, times(2)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + + unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); + + assertFalse(unprocessedManualEntities.isEmpty()); + assertEquals(unprocessedManualEntities.size(), 1); + assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); + assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%"); + assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance "); + assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + } + + + @Test + void testMultipleManualResizeRequests() { + + String pdfFile = "files/new/S4.pdf"; + + ManualRedactions manualRedactions = new ManualRedactions(); + + var aoelId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + + var cormsId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS"); + + var a9Id = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G"); + + manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); + manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); + manualRedactions.getEntriesToAdd().add(manualRedactionEntry); + AnalyzeRequest request = uploadFileToStorage(pdfFile); + request.setManualRedactions(manualRedactions); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + List positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build()); + List positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build()); + List positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build()); + ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL"); + ManualResizeRedaction manualResizeRedaction2 = prepareManualSizeRedaction(cormsId, positions2, "CoRMS proposed"); + ManualResizeRedaction manualResizeRedaction3 = prepareManualSizeRedaction(a9Id, positions3, "metolachlor in A9396G"); + + request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction, manualResizeRedaction2, manualResizeRedaction3)).build()); + unprocessedChangesService.analyseSurroundingText(request); + + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + + List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); + + assertFalse(unprocessedManualEntities.isEmpty()); + assertEquals(unprocessedManualEntities.size(), 3); + + var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst(); + assertTrue(resizedAoel.isPresent()); + assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of"); + assertEquals(resizedAoel.get().getTextBefore(), "is below the "); + assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); + assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth()); + assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight()); + + var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst(); + assertTrue(cormsResized.isPresent()); + assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of"); + assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore "); + assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the"); + assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX()); + assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY()); + assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth()); + assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight()); + + var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst(); + assertTrue(a9Resized.isPresent()); + assertEquals(a9Resized.get().getTextAfter(), " were obtained from"); + assertEquals(a9Resized.get().getTextBefore(), "data for S"); + assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); + assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX()); + assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY()); + assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth()); + assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight()); + } + + + @Test + void testManualResizeIfMultipleValuesInTheSameNode() { + + String pdfFile = "files/new/S4.pdf"; + + ManualRedactions manualRedactions = new ManualRedactions(); + var aoelId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + + manualRedactions.getEntriesToAdd().add(manualRedactionEntry); + AnalyzeRequest request = uploadFileToStorage(pdfFile); + request.setManualRedactions(manualRedactions); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + List positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build()); + ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL"); + + request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); + unprocessedChangesService.analyseSurroundingText(request); + + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + + List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); + + assertFalse(unprocessedManualEntities.isEmpty()); + assertEquals(unprocessedManualEntities.size(), 1); + assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); + assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of"); + assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the "); + assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + } + + + @Test + void testManualResizeInADifferentSection() { + + String pdfFile = "files/new/S4.pdf"; + + ManualRedactions manualRedactions = new ManualRedactions(); + var aoelId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + + manualRedactions.getEntriesToAdd().add(manualRedactionEntry); + AnalyzeRequest request = uploadFileToStorage(pdfFile); + request.setManualRedactions(manualRedactions); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + List positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build()); + ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL"); + + request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); + unprocessedChangesService.analyseSurroundingText(request); + + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + + List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); + + assertFalse(unprocessedManualEntities.isEmpty()); + assertEquals(unprocessedManualEntities.size(), 1); + assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); + assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same"); + assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an "); + assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the"); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + } + + + @Test + void testManualResizeNotFound() { + + String pdfFile = "files/new/S4.pdf"; + + ManualRedactions manualRedactions = new ManualRedactions(); + var aoelId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + + manualRedactions.getEntriesToAdd().add(manualRedactionEntry); + AnalyzeRequest request = uploadFileToStorage(pdfFile); + request.setManualRedactions(manualRedactions); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + List positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build()); + ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist"); + + request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); + unprocessedChangesService.analyseSurroundingText(request); + + verify(rabbitTemplate, times(1)).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture()); + + List unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities(); + + assertTrue(unprocessedManualEntities.isEmpty()); + } + + + private static ManualResizeRedaction prepareManualSizeRedaction(String id, List positions, String value) { + + ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction(); + manualResizeRedaction.setAnnotationId(id); + manualResizeRedaction.setPositions(positions); + manualResizeRedaction.setUpdateDictionary(false); + manualResizeRedaction.setAddToAllDossiers(false); + manualResizeRedaction.setValue(value); + return manualResizeRedaction; + } + + + private static ManualRedactionEntry prepareManualRedactionEntry(String id, List positions, String value) { + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(id); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setStatus(AnnotationStatus.APPROVED); + manualRedactionEntry.setType("CBI_author"); + manualRedactionEntry.setValue(value); + manualRedactionEntry.setReason("Manual Redaction"); + manualRedactionEntry.setPositions(positions); + return manualRedactionEntry; } } diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java index 791fd24e..2c835963 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java @@ -26,6 +26,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; import lombok.SneakyThrows; +@SuppressWarnings("PMD") class RuleFileFactoryTest { @Test diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java index eadcf3b0..c63ff51c 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java @@ -28,6 +28,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; import lombok.SneakyThrows; +@SuppressWarnings("PMD") class OldRulesParserTest { @Test