diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index fcf8b9e5..cbcbcd3d 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.70.0" val jacksonVersion = "2.15.2" val droolsVersion = "8.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.182.0" +val persistenceServiceVersion = "2.187.0" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java index 724812ca..78116c41 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java @@ -10,6 +10,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -34,11 +36,11 @@ public class Entity { boolean imported; + SemanticNode containingNode; String section; float[] color; List positions; - int sectionNumber; String textBefore; String textAfter; @@ -65,7 +67,7 @@ public class Entity { Set importedRedactionIntersections; - public static Entity fromEntityLogEntry(EntityLogEntry e) { + public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) { return Entity.builder() .id(e.getId()) @@ -79,8 +81,7 @@ public class Entity { .imported(e.isImported()) .section(e.getSection()) .color(e.getColor()) - .positions(e.getPositions()) - .sectionNumber(e.getSectionNumber()) + .positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode()) .textBefore(e.getTextBefore()) .textAfter(e.getTextAfter()) .startOffset(e.getStartOffset()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java index 2d8f4726..c5e75340 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java @@ -249,7 +249,7 @@ public interface SemanticNode { * Ignores Entity with ignored == true or removed == true. * * @param types an array of strings representing the types of entities to check for - * @return true, if this SemanticNode has at least one Entity of any of the provided types + * @return true, if this SemanticNode has at least one Entity of the provided types */ default boolean hasEntitiesOfAnyType(String... types) { @@ -455,7 +455,7 @@ public interface SemanticNode { * @param y the lower left corner Y value * @param w width * @param h height - * @param pageNumber the pagenumber of the rectangle + * @param pageNumber the pageNumber of the rectangle * @return true if intersects, false otherwise */ default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) { @@ -607,7 +607,7 @@ public interface SemanticNode { private Map getBBoxFromLeafTextBlock(Map bBoxPerPage) { Map> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage)); - atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atbs))); + atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks))); return bBoxPerPage; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index bc6b3d46..7fbff336 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -122,7 +122,7 @@ public class AnalyzeService { dictionary.getVersion(), kieWrapperEntityRules.rulesVersion()); - return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), + return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document, redactionLog, document.getNumberOfPages(), dictionary.getVersion(), @@ -165,7 +165,7 @@ public class AnalyzeService { entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, dictionaryIncrement.getDictionaryVersion(), analyzeRequest.getDossierTemplateId(), - false), + false), document, previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -206,7 +206,7 @@ public class AnalyzeService { return finalizeAnalysis(analyzeRequest, startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), - entityLogChanges, + entityLogChanges, document, redactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -240,7 +240,7 @@ public class AnalyzeService { } - private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, + private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document, RedactionLog redactionLog, int numberOfPages, DictionaryVersion dictionaryVersion, @@ -254,7 +254,7 @@ public class AnalyzeService { log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (entityLogChanges.isHasChanges() || !isReanalysis) { - computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges, dictionaryVersion); + computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion); } log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -284,7 +284,7 @@ public class AnalyzeService { private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, - KieWrapper kieWrapperComponentRules, + KieWrapper kieWrapperComponentRules, Document document, Set addedFileAttributes, EntityLogChanges entityLogChanges, DictionaryVersion dictionaryVersion) { @@ -294,7 +294,7 @@ public class AnalyzeService { } List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), - entityLogChanges.getEntityLog(), + entityLogChanges.getEntityLog(), document, addedFileAttributes.stream().toList()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java index 339c9156..4394d8a8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -6,8 +6,9 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentValue; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.EntityReference; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; @@ -18,36 +19,35 @@ public class ComponentLogCreatorService { public ComponentLog buildComponentLog(int analysisNumber, List components, long componentRulesVersion) { - List componentLogComponents = components.stream() + List componentLogComponents = components.stream() .collect(Collectors.groupingBy(Component::getName, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) .entrySet() - .stream() - .map(entry -> new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.Component(entry.getKey(), entry.getValue())) + .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue())) .toList(); return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents); } - private ComponentValue buildComponentLogEntry(Component component) { + private ComponentLogEntryValue buildComponentLogEntry(Component component) { - return ComponentValue.builder() + return ComponentLogEntryValue.builder() .value(component.getValue()).originalValue(component.getValue()) .componentRuleId(component.getMatchedRule().toString()) .valueDescription(component.getValueDescription()) - .entityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList())) + .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.start()).toList())) .build(); } - private List toComponentEntityReferences(List references) { + private List toComponentEntityReferences(List references) { return references.stream().map(this::toComponentEntityReference).toList(); } - private EntityReference toComponentEntityReference(Entity entity) { + private ComponentLogEntityReference toComponentEntityReference(Entity entity) { - return EntityReference.builder().id(entity.getId()) + return ComponentLogEntityReference.builder().id(entity.getId()) .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule()) .type(entity.getType()) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 1081b4f2..b78f5afc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -86,12 +86,15 @@ public class EntityLogCreatorService { public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, Document document, List notFoundManualRedactionEntries, - EntityLog previousEntityLog, Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { + EntityLog previousEntityLog, + Set sectionsToReanalyseIds, + DictionaryVersion dictionaryVersion) { List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); List previousEntries = previousEntityLog.getEntityLogEntry() .stream() - .filter(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)) + .filter(entry -> (entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType() + .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)) .toList(); previousEntityLog.getEntityLogEntry().removeAll(previousEntries); boolean hasChanges = entityChangeLogService.computeChanges(previousEntries, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); @@ -140,8 +143,7 @@ public class EntityLogCreatorService { private List createEntityLogEntriesFromActiveEntities(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { List entries = new ArrayList<>(); - document.getEntities() - .stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active) + document.getEntities().stream().filter(EntityLogCreatorService::isEntityOrRecommendationType).filter(IEntity::active) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); document.streamAllImages().filter(IEntity::active).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); notFoundManualRedactionEntries.stream().filter(IEntity::active).forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); @@ -182,33 +184,30 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) { + public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { - Set referenceIds = new HashSet<>(); - entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); - int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); - boolean isHint = isHint(entity.getType(), dossierTemplateId); + String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); + boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); return EntityLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) - .reason(entity.buildReasonWithManualChangeDescriptions()) - .legalBasis(entity.legalBasis()) - .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) - .type(entity.getType()) - .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) - .sectionNumber(sectionNumber) - .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(entity.isDictionaryEntry()) - .textAfter(entity.getTextAfter()) - .textBefore(entity.getTextBefore()) - .startOffset(entity.getTextRange().start()) - .endOffset(entity.getTextRange().end()) - .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) - .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) - .reference(referenceIds) - .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) - .state(buildEntryState(entity)) - .entryType(buildEntryType(entity, isHint)) + .id(image.getId()) + .value(image.value()) + .color(getColor(imageType, dossierTemplateId, image.applied())) + .value(image.value()) + .type(imageType) + .reason(image.buildReasonWithManualChangeDescriptions()) + .legalBasis(image.legalBasis()) + .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(false) + .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) + .containingNodeId(image.getTreeId()) + .closestHeadline(image.getHeadline().getTextBlock().getSearchText()) + .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) + .imageHasTransparency(image.isTransparent()) + .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) + .state(buildEntryState(image)) + .entryType(buildEntryType(image, isHint)) .build(); + } @@ -226,8 +225,9 @@ public class EntityLogCreatorService { .state(buildEntryState(manualEntity)) .entryType(buildEntryType(manualEntity, isHint)) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) - .sectionNumber(0) - .matchedRule("ManualRedaction") + .containingNodeId(Collections.emptyList()) + .closestHeadline("") + .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) .dictionaryEntry(manualEntity.isDictionaryEntry()) .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) .textAfter("") @@ -242,29 +242,34 @@ public class EntityLogCreatorService { } - public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { + private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) { - String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); - boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); + Set referenceIds = new HashSet<>(); + entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); + int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); + boolean isHint = isHint(entity.getType(), dossierTemplateId); return EntityLogEntry.builder() - .id(image.getId()) - .value(image.value()) - .color(getColor(imageType, dossierTemplateId, image.applied())) - .value(image.value()) - .type(imageType) - .reason(image.buildReasonWithManualChangeDescriptions()) - .legalBasis(image.legalBasis()) - .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(false) - .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) - .sectionNumber(image.getTreeId().get(0)) - .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) - .imageHasTransparency(image.isTransparent()) - .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) - .state(buildEntryState(image)) - .entryType(buildEntryType(image, isHint)) + .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) + .reason(entity.buildReasonWithManualChangeDescriptions()) + .legalBasis(entity.legalBasis()) + .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) + .type(entity.getType()) + .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) + .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) + .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) + .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(entity.isDictionaryEntry()) + .textAfter(entity.getTextAfter()) + .textBefore(entity.getTextBefore()) + .startOffset(entity.getTextRange().start()) + .endOffset(entity.getTextRange().end()) + .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) + .reference(referenceIds) + .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) + .state(buildEntryState(entity)) + .entryType(buildEntryType(entity, isHint)) .build(); - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index 9fbe252d..6b770be4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -9,7 +9,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.OptionalInt; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -17,6 +17,9 @@ import org.kie.api.runtime.KieSession; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; import com.iqser.red.service.redaction.v1.server.utils.DateConverter; @@ -32,6 +35,14 @@ public class ComponentCreationService { Set referencedEntities = new HashSet<>(); + /** + * Finds the first value from the collection of entities and creates a component from it. If no value is found, the fallback value is used instead. + * + * @param ruleIdentifier the identifier for the rule + * @param name the name of the operation + * @param entities the collection of entities to search for the first value + * @param fallback the value to be returned if no value is found in the collection + */ public void firstOrElse(String ruleIdentifier, String name, Collection entities, String fallback) { String valueDescription = String.format("First found value or else '%s'", fallback); @@ -40,24 +51,45 @@ public class ComponentCreationService { } + /** + * Creates a new component with the given parameters and inserts it into the kieSession. + * + * @param ruleIdentifier The rule identifier for the component. + * @param name The name of the component. + * @param value The value of the component. + * @param valueDescription The description of the value. + * @param references A collection of Entity objects that the component references. + */ public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection references) { referencedEntities.addAll(references); - kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name) - .value(value).valueDescription(valueDescription) + kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription) .references(new LinkedList<>(references)) .build()); } + /** + * Joins entity values, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joining(String ruleIdentifier, String name, Collection entities) { joining(ruleIdentifier, name, entities, ", "); } + /** + * Joins entity values, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joining(String ruleIdentifier, String name, Collection entities, String delimiter) { String valueDescription = String.format("Joining all values with '%s'", delimiter); @@ -66,12 +98,26 @@ public class ComponentCreationService { } + /** + * Joins entity values from the first section entities appear in, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities) { joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", "); } + /** + * Joins unique entity values from the first section entities appear in, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { List entitiesFromFirstSection = findEntitiesFromFirstSection(entities); @@ -81,21 +127,35 @@ public class ComponentCreationService { private static List findEntitiesFromFirstSection(Collection entities) { - var entitiesBySection = entities.stream().collect(Collectors.groupingBy(Entity::getSectionNumber)); - OptionalInt firstSection = entitiesBySection.keySet().stream().mapToInt(Integer::intValue).min(); + var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); + Optional firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first()); if (firstSection.isEmpty()) { return Collections.emptyList(); } - return entitiesBySection.get(firstSection.getAsInt()); + return entitiesBySection.get(firstSection.get()); } + /** + * Joins unique entity values from the first section entities appear in, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities) { joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", "); } + /** + * Joins entity values from the first section entities appear in, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { List entitiesFromFirstSection = findEntitiesFromFirstSection(entities); @@ -103,29 +163,14 @@ public class ComponentCreationService { } - private static List findEntitiesFromLongestSection(Collection entities) { - - var entitiesBySection = entities.stream().collect(Collectors.groupingBy(Entity::getSectionNumber)); - OptionalInt longestSection = entitiesBySection.entrySet() - .stream() - .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()) - .mapToInt(Map.Entry::getKey) - .findFirst(); - - if (longestSection.isEmpty()) { - return Collections.emptyList(); - } - - return entitiesBySection.get(longestSection.getAsInt()); - } - - - private static int getTotalLengthOfEntities(Map.Entry> entry) { - - return entry.getValue().stream().mapToInt(Entity::getLength).sum(); - } - - + /** + * Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the joining operation + * @param entities the collection of entities + * @param delimiter the delimiter to use for joining the values + */ public void joiningUnique(String ruleIdentifier, String name, Collection entities, String delimiter) { String valueDescription = String.format("Joining all values with '%s'", delimiter); @@ -134,6 +179,26 @@ public class ComponentCreationService { } + /** + * Joins entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ + public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection entities) { + + joiningFromLongestSectionOnly(ruleIdentifier, name, entities, ", "); + } + + + /** + * Joins entity values from the section with the longest combined entity values only, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { List entitiesFromLongestSection = findEntitiesFromLongestSection(entities); @@ -141,6 +206,49 @@ public class ComponentCreationService { } + private static List findEntitiesFromLongestSection(Collection entities) { + + var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); + Optional longestSection = entitiesBySection.entrySet() + .stream() + .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()).map(Map.Entry::getKey) + .findFirst(); + + if (longestSection.isEmpty()) { + return Collections.emptyList(); + } + + return entitiesBySection.get(longestSection.get()); + } + + + private static int getTotalLengthOfEntities(Map.Entry> entry) { + + return entry.getValue().stream().mapToInt(Entity::getLength).sum(); + } + + + /** + * Joins unique entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + */ + public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection entities) { + + joiningUniqueFromLongestSectionOnly(ruleIdentifier, name, entities, ", "); + } + + + /** + * Joins unique entity values from the section with the longest combined entity values only, and creates a component from the result. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities The collection of entities to process. + * @param delimiter The string delimiter to join the entities. + */ public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection entities, String delimiter) { List entitiesFromLongestSection = findEntitiesFromLongestSection(entities); @@ -148,40 +256,88 @@ public class ComponentCreationService { } + /** + * Joins the names of the entities in the provided collection, separated by a comma, + * and creates a component from the result. + * + * @param ruleIdentifier The unique identifier of the rule being applied. + * @param name The name to which the joined result will be set. + * @param entities The collection of entities whose names will be joined. + */ public void joiningUnique(String ruleIdentifier, String name, Collection entities) { joiningUnique(ruleIdentifier, name, entities, ", "); } + /** + * Computes the number of unique values in the collection of entities and creates a component with the result. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the record + * @param entities the collection of entities to compute unique values from + */ + public void uniqueValueCount(String ruleIdentifier, String name, Collection entities) { + + long count = entities.stream().map(Entity::getValue).distinct().count(); + create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities); + } + + + /** + * Creates a component for each sentence in the collection of entities. + * + * @param ruleIdentifier The identifier of the rule. + * @param name The name of the entity. + * @param entities A collection of Entity objects. + */ public void asSentences(String ruleIdentifier, String name, Collection entities) { if (entities.isEmpty()) { return; } - for (Entity entity : entities) { BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH); iterator.setText(entity.getValue()); int start = iterator.first(); for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) { - create(ruleIdentifier, name, entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), "Split into sentences", entity); + create(ruleIdentifier, + name, + entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), + String.format("Values of type '%s' as sentences", entity.getType()), + entity); } } } + /** + * Creates a new component with the given rule identifier, name, value, value description, and reference. + * The component is built using the provided parameters and inserted into the knowledge session. + * The reference is added to the referencedEntities list. + * + * @param ruleIdentifier The identifier of the rule for the component. + * @param name The name of the component. + * @param value The value of the component. + * @param valueDescription The description of the value. + * @param reference The reference entity for the component. + */ public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) { referencedEntities.add(reference); List referenceList = new LinkedList<>(); referenceList.add(reference); - kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription) - .references(referenceList) - .build()); + kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value) + .valueDescription(valueDescription).references(referenceList).build()); } + /** + * Creates components for unmapped entities. + * + * @param ruleIdentifier The identifier of the rule being applied. + * @param entities The collection of entities to create components for. + */ public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection entities) { entities.stream() @@ -190,30 +346,109 @@ public class ComponentCreationService { } + /** + * Converts entity values to the 'dd/MM/yyyy' format and joins them with ', '. If the value could not be parsed as a date, it will be created as is. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the entity + * @param entities the collection of entities + */ public void convertDates(String ruleIdentifier, String name, Collection entities) { convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy"); } + /** + * Converts entity values to provided format and joins them with ', '. If the value could not be parsed as a date, it will be created as is. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the entity + * @param entities the collection of entities + * @param resultFormat the desired format for the converted dates + */ public void convertDates(String ruleIdentifier, String name, Collection entities, String resultFormat) { - String valueDescription = "Convert values of type to dd/MM/yyyy joined with ', '"; + String valueDescription = String.format("Convert values of type to %s joined with ', '", resultFormat); String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", ")); create(ruleIdentifier, name, date, valueDescription, entities); } + /** + * Joins values from entities that are in the same table row. If entities are not in a table cell they are added as a single component. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the entity + * @param entities the collection of entities + */ + public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection entities) { + + String types = entities.stream().map(Entity::getType).distinct().collect(Collectors.joining()); + String valueDescription = String.format("Combine values of %s that are in same table row", types); + Map, List> entitiesPerTable = entities.stream().collect(Collectors.groupingBy(this::getFirstTable)); + entitiesPerTable.forEach((optionalTable, groupedEntities) -> { + if (optionalTable.isEmpty()) { + groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity)); + } + + groupedEntities.stream() + .filter(entity -> !(entity.getContainingNode() instanceof TableCell)) + .forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity)); + + groupedEntities.stream() + .filter(entity -> entity.getContainingNode() instanceof TableCell) + .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())) + .forEach((row, entitiesInSameRow) -> create(ruleIdentifier, + name, + entities.stream().map(Entity::getValue).collect(Collectors.joining(", ")), + valueDescription, + entitiesInSameRow)); + }); + } + + + private Optional getFirstTable(Entity entity) { + + SemanticNode node = entity.getContainingNode(); + while (!(node instanceof Table)) { + if (!node.hasParent()) { + return Optional.empty(); + } + node = node.getParent(); + } + + return Optional.of((Table) node); + } + + + /** + * Creates a new component with the given rule identifier, name, value, and value description. + * If the component is part of a table, it also takes a list of entities that belong to the same table row. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the entity + * @param value the value of the component + * @param valueDescription the description of the value + */ public void create(String ruleIdentifier, String name, String value, String valueDescription) { create(ruleIdentifier, name, value, valueDescription, Collections.emptyList()); } + /** + * Creates a new component with the given rule identifier, name, and value. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the entity + * @param value the value of the component + */ public void create(String ruleIdentifier, String name, String value) { kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name) - .value(value).valueDescription("") + .value(value) + .valueDescription("") .references(Collections.emptyList()) .build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index 3eeab748..18fd4fd3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -47,7 +47,10 @@ public class SectionFinderService { Set sectionsToReanalyse = new HashSet<>(); for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { - sectionsToReanalyse.add(entry.getSectionNumber()); + if (entry.getContainingNodeId().isEmpty()) { + continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis. + } + sectionsToReanalyse.add(entry.getContainingNodeId().get(0)); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java new file mode 100644 index 00000000..165d8c15 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java @@ -0,0 +1,46 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.Comparator; +import java.util.List; + +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +public abstract class SemanticNodeComparators implements Comparator { + + public static SemanticNodeComparators first() { + + return new FirstSemanticNode(); + } + + public static class FirstSemanticNode extends SemanticNodeComparators { + + @Override + public int compare(SemanticNode semanticNode, SemanticNode otherSemanticNode) { + + List treeId = semanticNode.getTreeId(); + List otherTreeId = otherSemanticNode.getTreeId(); + + int prefixLength = Math.min(treeId.size(), otherTreeId.size()); + // Compare ids one by one + for (int i = 0; i < prefixLength; i++) { + int id1 = treeId.get(i); + int id2 = otherTreeId.get(i); + + // If ids are different, return the comparison result + if (id1 != id2) { + return Integer.compare(id1, id2); + } + } + + // If all prefix ids are equal, shorter treeId is first + if (treeId.size() != otherTreeId.size()) { + return Integer.compare(treeId.size(), otherTreeId.size()); + } + + // Lists are equal + return 0; + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index f25ad1d4..76a9dfa2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -7,8 +7,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; import org.kie.api.runtime.rule.QueryResults; @@ -16,11 +14,14 @@ import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -36,13 +37,13 @@ public class ComponentDroolsExecutionService { RedactionServiceSettings settings; - public List executeRules(KieContainer kieContainer, EntityLog entityLog, List fileAttributes) { + public List executeRules(KieContainer kieContainer, EntityLog entityLog, Document document, List fileAttributes) { KieSession kieSession = kieContainer.newKieSession(); ComponentCreationService componentCreationService = new ComponentCreationService(kieSession); kieSession.setGlobal("componentCreationService", componentCreationService); - entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert); + entityLog.getEntityLogEntry().stream().map(entry -> Entity.fromEntityLogEntry(entry, document)).forEach(kieSession::insert); fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index dfedeb21..31f2e75f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -113,7 +113,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow(); + var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow(); assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); @@ -126,7 +126,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow(); + var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow(); assertEquals(EntryState.APPLIED, asyaLyon2.getState()); @@ -141,13 +141,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { } - private Stream findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, int sectionNumber) { + private Stream findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, List sectionNumber) { return redactionLog.getEntityLogEntry() .stream() .filter(entry -> entry.getType().equals(type)) - .filter(entry -> entry.getValue().equals(value)) - .filter(entry -> entry.getSectionNumber() == sectionNumber); + .filter(entry -> entry.getValue().equals(value)).filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index c5784b7c..79933126 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -589,11 +589,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } - public void testRedactionLogAndEntityLogEquality() { - - } - - @Test public void testRemovePublishedInformations() throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 9cfcc375..9a6545f8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server; import static java.util.Map.entry; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.io.BufferedReader; @@ -81,7 +80,6 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; @@ -473,7 +471,7 @@ public class RulesTest { assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)); assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection()); assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor()); - assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber()); + assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId().get(0)); assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore()); assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter()); assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java new file mode 100644 index 00000000..f4c315fb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java @@ -0,0 +1,67 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +class SemanticNodeComparatorsTest { + + @Test + public void testFirstSemanticNode() { + + var node = new Section(List.of(0, 1), null, null, null); + var otherNode = new Section(List.of(0, 2), null, null, null); + List list = new ArrayList<>(); + list.add(otherNode); + list.add(node); + list.sort(SemanticNodeComparators.first()); + assertEquals(node, list.get(0)); + } + + + @Test + public void testFirstSemanticNode2() { + + var node = new Section(Collections.emptyList(), null, null, null); + var otherNode = new Section(List.of(0, 2), null, null, null); + List list = new ArrayList<>(); + list.add(otherNode); + list.add(node); + list.sort(SemanticNodeComparators.first()); + assertEquals(node, list.get(0)); + } + + + @Test + public void testFirstSemanticNode3() { + + var node = new Section(List.of(1, 5, 8), null, null, null); + var otherNode = new Section(List.of(0, 2), null, null, null); + List list = new ArrayList<>(); + list.add(otherNode); + list.add(node); + list.sort(SemanticNodeComparators.first()); + assertEquals(otherNode, list.get(0)); + } + + + @Test + public void testFirstSemanticNode4() { + + var node = new Section(List.of(1, 5, 8), null, null, null); + var otherNode = new Section(List.of(1, 5, 9), null, null, null); + List list = new ArrayList<>(); + list.add(otherNode); + list.add(node); + list.sort(SemanticNodeComparators.first()); + assertEquals(node, list.get(0)); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl index 21eb2477..b9120f67 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -49,7 +49,7 @@ declare GuidelineMapping rule "StudyTitle.0.0: Study Title" when - $titleCandidates: List() from collect (Entity(type == "study_title")) + $titleCandidates: List() from collect (Entity(type == "title")) then componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!"); end @@ -180,38 +180,331 @@ rule "TestGuideline.2.0: Test Guideline 2" end -rule "DefaultComponents.6.0: Experimental Starting Date" +rule "StartDate.0.0: Experimental Starting Date" when $startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date")) then - componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates); + componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates); end -rule "DefaultComponents.7.0: Experimental Completion Date" +rule "CompletionDate.0.0: Experimental Completion Date" when $endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date")) then - componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates); + componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates); end -rule "DefaultComponents.8.0: Certificate of analysis batch identification" +rule "AnalysisCertificate.0.0: Certificate of analysis batch identification" when $batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number")) then - componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers); + componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Batch_Number", $batchNumbers); end rule "StudyConclusion.0.0: Study conclusion in first found section" when - FileAttribute(label == "oecd_number", value == "425" || value == "430") + $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") + FileAttribute(label == "oecd_number", value == $oecdNumber) $studyConclusions: List() from collect(Entity(type == "study_conclusion")) then componentCreationService.joiningUniqueFromFirstSectionOnly("Study_Conclusion.0.0", "Study_Conclusion", $studyConclusions); end +rule "GuidelineDeviation.0.0: Guideline deviation as sentences" + when + $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation")) + then + componentCreationService.asSentences("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations); + end +rule "Species.0.0: First found species" + when + $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $species: List() from collect (Entity(type == "species")) + then + componentCreationService.firstOrElse("Species.0.0", "Species", $species, ""); + end + +rule "Strain.0.0: First found strain" + when + $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $strain: List() from collect (Entity(type == "strain")) + then + componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, ""); + end + +rule "Conclusion.0.0: Unique values of Conclusion LD50" + when + $oecdNumber: String() from List.of("402", "403", "425", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $conclusions: List() from collect (Entity(type == "ld50_value")) + then + componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions, ""); + end + +rule "Conclusion0.1.0: Greater than found" + when + $oecdNumber: String() from List.of("402", "403", "425", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater")) + then + componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater Than", "\"Greater than\" value found", $conclusions); + end + +rule "Conclusion.1.1: Greater than not found" + when + $oecdNumber: String() from List.of("402", "403", "425", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + not Entity(type == "ld50_greater") + then + componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No \"Greater than\" value found"); + end + +rule "Conclusion.2.0: Minimum confidence as unique values" + when + $oecdNumber: String() from List.of("402", "403", "425", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $conclusions: List() from collect (Entity(type == "confidence_minimal")) + then + componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions); + end + +rule "Conclusion.3.0: Maximum confidence as unique values" + when + $oecdNumber: String() from List.of("402", "403", "425", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $conclusions: List() from collect (Entity(type == "confidence_maximal")) + then + componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions); + end + +rule "Necropsy.0.0: Necropsy findings from longest section" + when + FileAttribute(label == "oecd_number", value == "402") + $necropsies: List() from collect (Entity(type == "necropsy_findings")) + then + componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies); + end + +rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block" + when + FileAttribute(label == "oecd_number", value == "402") + $dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)")) + then + componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " "); + end + +rule "Necropsy.2.0: Necropsy findings as one block" + when + $oecdNumber: String() from List.of("403", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $necropsies: List() from collect (Entity(type == "necropsy_findings")) + then + componentCreationService.joining("Necropsy.2.0", "Necropsy_Findings", $necropsies, " "); + end + +rule "Necropsy.3.0: Conducted with 4 hours of exposure as one block" + when + $oecdNumber: String() from List.of("403", "436") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $exposures: List() from collect (Entity(type == "4h_exposure")) + then + componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " "); + end + +rule "StudyDesign.0.0: Study design as one block" + when + $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $studyDesigns: List() from collect (Entity(type == "study_design")) + then + componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " "); + end + +rule "Results.0.0: Results and conclusions as joined values" + when + $oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $results: List() from collect (Entity(type == "results_and_conclusion")) + then + componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " "); + end + +rule "WeightBehavior.0.0: Weight change behavior as sentences" + when + FileAttribute(label == "oecd_number", value == "402") + $weightChanges: List() from collect (Entity(type == "weight_behavior_changes")) + then + componentCreationService.asSentences("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges); + end + +rule "MortalityStatement.0.0: Mortality statements as one block" + when + FileAttribute(label == "oecd_number", value == "402") + $mortalityStatements: List() from collect (Entity(type == "mortality_statement")) + then + componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " "); + end + +rule "ClinicalObservations.0.0: Clinical observations as sentences" + when + FileAttribute(label == "oecd_number", value == "403") + $observations: List() from collect (Entity(type == "clinical_observations")) + then + componentCreationService.asSentences("MortalityStatement.0.0", "Clinical_Observations", $observations); + end + +rule "BodyWeight.0.0: Bodyweight changes as sentences" + when + FileAttribute(label == "oecd_number", value == "403") + $weightChanges: List() from collect (Entity(type == "bodyweight_changes")) + then + componentCreationService.asSentences("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges); + end + +rule "Detailing.0.0: Detailing of reported changes as one block" + when + $oecdNumber: String() from List.of("404", "405") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $detailings: List() from collect (Entity(type == "detailing")) + then + componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " "); + end + +rule "Sex.0.0: Male sex found" + when + $oecdNumber: String() from List.of("405", "429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $males: List(!isEmpty) from collect (Entity(type == "sex", (value == "male" || value == "males"))) + then + componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males); + end + +rule "Sex.1.0: Female sex found" + when + $oecdNumber: String() from List.of("405", "429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $females: List(!isEmpty) from collect (Entity(type == "sex", (value == "female" || value == "females"))) + then + componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females); + end + +rule "NumberOfAnimals.0.0: Number of animals found" + when + $oecdNumber: String() from List.of("405", "429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $numberOfAnimals: Entity(type == "number_of_animals") + then + componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals); + end + +rule "NumberOfAnimals.1.0: Count unique occurences of animals" + when + $oecdNumber: String() from List.of("405", "429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + not Entity(type == "number_of_animals") + $animals: List() from collect (Entity(type == "animal_number")) + then + componentCreationService.uniqueValueCount("NumberOfAnimals.1.0", "Number_of_Animals", $animals); + end + +rule "ClinicalSigns.0.0: Clinical signs as sentences" + when + $oecdNumber: String() from List.of("425") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $clinicalSigns: List() from collect (Entity(type == "clinical_signs")) + then + componentCreationService.asSentences("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns); + end + +rule "DoseMortality.0.0: Dose mortality as sentences" + when + $oecdNumber: String() from List.of("425") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose")) + then + componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities); + end + +rule "Mortality.0.0: Mortality as one block" + when + $oecdNumber: String() from List.of("425") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $mortalities: List() from collect (Entity(type == "mortality")) + then + componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " "); + end + +rule "Dosages.0.0: First found value of Dosages" + when + $oecdNumber: String() from List.of("425") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $mortalities: List() from collect (Entity(type == "mortality")) + then + componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, ""); + end + +rule "PrelimResults.0.0: Preliminary test results as sentences" + when + $oecdNumber: String() from List.of("429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $results: List() from collect (Entity(type == "preliminary_test_results")) + then + componentCreationService.asSentences("PrelimResults.0.0", "Preliminary_Test_Results", $results); + end + +rule "TestResults.0.0: Test results as one block" + when + $oecdNumber: String() from List.of("429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $results: List() from collect (Entity(type == "test_results")) + then + componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " "); + end + +rule "PositiveControl.0.0: Was the definitive study conducted with positive control" + when + $oecdNumber: String() from List.of("429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $results: List() from collect (Entity(type == "positive_control")) + then + componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " "); + end + +rule "MainResults.0.0: Was the definitive study conducted with positive control" + when + $oecdNumber: String() from List.of("429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $results: List() from collect (Entity(type == "results_(main_study)")) + then + componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " "); + end + +rule "UsedApproach.0.0: Was the definitive study conducted with positive control" + when + $oecdNumber: String() from List.of("429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + $results: List(!isEmpty()) from collect (Entity(type == "approach_used")) + then + componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results); + end + +rule "UsedApproach.1.0: Was the definitive study conducted with positive control" + when + $oecdNumber: String() from List.of("429") + FileAttribute(label == "oecd_number", value == $oecdNumber) + not Entity(type == "approach_used") + then + componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'"); + end + +/* rule "DefaultComponents.999.0: Create components for all unmapped entities." salience -999 when @@ -220,7 +513,7 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities." componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities); end - +*/ //------------------------------------ Component merging rules ------------------------------------