RED-6369: Rules Refactor

* minor renaming
* added NerEntityAdapter to include the Ner Entities
* moved combine logic outside of rules

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java

@@ -262,7 +262,7 @@ public class DocumentGraphFactory {
                 .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                 .imageType(image.getImageType())
                 .position(position)
-                .transparency(image.isHasTransparency())
+                .transparent(image.isHasTransparency())
                 .page(page)
                 .tableOfContents(context.tableOfContents())
                 .build();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java

@@ -33,7 +33,7 @@ public class ImageNode implements SemanticNode {
     String id;
 
     ImageType imageType;
-    boolean transparency;
+    boolean transparent;
     Rectangle2D position;
 
     boolean redaction;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
+import static java.lang.String.format;
+
 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
 import java.util.List;
@@ -47,6 +49,20 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Each AtomicTextBlock is assigned a page, so to get the pages for this boundary, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock.
+     *
+     * @return Set of PageNodes this node appears on.
+     */
+    default Set<PageNode> getPages(Boundary boundary) {
+
+        if (!getBoundary().contains(boundary)) {
+            throw new IllegalArgumentException(format("%s which was used to query foir pages is not contained in the %s of this node!", getBoundary(), boundary));
+        }
+        return buildTextBlock().getPages(boundary);
+    }
+
+
     /**
      * @return the TableOfContents of the ClassificationDocument this node belongs to
      */

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java

@@ -57,6 +57,15 @@ public interface TextBlock extends CharSequence {
     }
 
 
+    default Set<PageNode> getPages(Boundary boundary) {
+
+        return getAtomicTextBlocks().stream()
+                .filter(atomicTextBlock -> atomicTextBlock.getBoundary().intersects(boundary))
+                .map(AtomicTextBlock::getPage)
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+
     default int indexOf(String searchTerm, int startOffset) {
 
         int start = getSearchText().indexOf(searchTerm, startOffset - getBoundary().start());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java

@@ -17,7 +17,7 @@ public class PropertiesMapper {
 
         Map<String, String> properties = new HashMap<>();
         properties.put("imageType", image.getImageType().toString());
-        properties.put("transparency", String.valueOf(image.isTransparency()));
+        properties.put("transparency", String.valueOf(image.isTransparent()));
         properties.put("position", RectangleTransformations.toString(image.getPosition()));
         properties.put("id", image.getId());
         return properties;
@@ -53,7 +53,7 @@ public class PropertiesMapper {
     public static void parseImageProperties(Map<String, String> properties, ImageNode.ImageNodeBuilder builder) {
 
         builder.imageType(parseImageType(properties.get("imageType")));
-        builder.transparency(Boolean.parseBoolean(properties.get("transparency")));
+        builder.transparent(Boolean.parseBoolean(properties.get("transparency")));
         builder.position(parseRectangle2D(properties.get("position")));
         builder.id(properties.get("id"));
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java

@@ -8,7 +8,6 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -148,35 +147,26 @@ public class EntityCreationService {
 
     public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
 
-        try {
+        tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity);
-            SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
-                    .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
-                    .findFirst()
-                    .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
-
-            containingNode.addThisToEntityIfIntersects(entity);
 
         TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock();
         entityEnrichmentService.enrichEntity(entity, textBlock);
 
         addToPages(entity);
-            addToNodeEntitySets(entity);
+        addEntityToNodeEntitySets(entity);
 
-        } catch (NoSuchElementException e) {
-            entity.removeFromGraph();
-        }
     }
 
 
-    private void addToPages(EntityNode entity) {
+    private static void addToPages(EntityNode entity) {
 
-        Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages();
+        Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary());
         entity.getPages().addAll(pages);
         pages.forEach(page -> page.getEntities().add(entity));
     }
 
 
-    private void addToNodeEntitySets(EntityNode entity) {
+    private static void addEntityToNodeEntitySets(EntityNode entity) {
 
         entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java

@@ -0,0 +1,146 @@
+package com.iqser.red.service.redaction.v1.server.redaction.adapter;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+public class NerEntitiesAdapter {
+
+    RedactionStorageService redactionStorageService;
+    RedactionServiceSettings redactionServiceSettings;
+
+    // imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition
+    private static final Set<String> CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE");
+    private static final Set<String> CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY");
+    private static final int MAX_DISTANCE_BETWEEN_PARTS = 20;
+    private static final int MIN_PART_MATCHES = 3;
+    private static final boolean ALLOW_DUPLICATES = false;
+
+
+    public List<EntityRecognitionEntity> getNerEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        } else {
+            nerEntities = new NerEntities();
+        }
+
+        addSectionStartOffsetToEachNerEntity(getStringStartOffsetsForMainSections(documentGraph), nerEntities);
+
+        List<EntityRecognitionEntity> entityRecognitionEntities = validateForCorrectOffsets(documentGraph, nerEntities);
+
+        List<EntityRecognitionEntity> cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList();
+        List<EntityRecognitionEntity> cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock());
+
+        return Stream.concat(cbiAuthors.stream(), cbiAddress.stream()).toList();
+    }
+
+
+    private List<EntityRecognitionEntity> combineToCBIAddressEntities(List<EntityRecognitionEntity> entityRecognitionEntities, TextBlock textBlock) {
+
+        Set<EntityRecognitionEntity> entitiesOfTypeCBIAddressParts = entityRecognitionEntities.stream()
+                .filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType()))
+                .collect(Collectors.toSet());
+
+        return entitiesOfTypeCBIAddressParts.stream()
+                .filter(entity -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(entity.getType()))
+                .map(entity -> findFollowingEntitiesOfTypeCBIAddressPart(entity, entitiesOfTypeCBIAddressParts))
+                .filter(entities -> entities.size() < MIN_PART_MATCHES)
+                .map(this::toContainingBoundary)
+                .distinct()
+                .map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address"))
+                .toList();
+    }
+
+
+    private List<EntityRecognitionEntity> validateForCorrectOffsets(DocumentGraph documentGraph, NerEntities nerEntities) {
+
+        return nerEntities.getData().values().stream().flatMap(Collection::stream).filter(nerEntity -> nerEntityOffsetMatches(nerEntity, documentGraph.buildTextBlock())).toList();
+    }
+
+
+    public Boundary toContainingBoundary(Collection<EntityRecognitionEntity> entityRecognitionEntities) {
+
+        int minStart = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).min().orElseThrow(IllegalArgumentException::new);
+        int maxEnd = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).max().orElseThrow(IllegalArgumentException::new);
+        return new Boundary(minStart, maxEnd);
+    }
+
+
+    public Set<EntityRecognitionEntity> findFollowingEntitiesOfTypeCBIAddressPart(EntityRecognitionEntity entityOfCBIAddressEssentialType,
+                                                                                  Set<EntityRecognitionEntity> entitiesOfTypeCBIAddressPart) {
+
+        Set<EntityRecognitionEntity> concurrentParts = new HashSet<>();
+        concurrentParts.add(entityOfCBIAddressEssentialType);
+
+        int currentNumberOfConcurrentParts;
+        do {
+            currentNumberOfConcurrentParts = concurrentParts.size();
+            concurrentParts.addAll(entitiesOfTypeCBIAddressPart.stream()
+                    .filter(entity -> entity.getStartOffset() >= toContainingBoundary(concurrentParts).end())
+                    .filter(entity -> offsetBetweenPartsIsLowerThanMax(toContainingBoundary(concurrentParts), entity))
+                    .filter(entity -> ALLOW_DUPLICATES || entityTypeInAlreadyFound(concurrentParts, entity.getType()))
+                    .collect(Collectors.toSet()));
+        } while (currentNumberOfConcurrentParts != concurrentParts.size());
+
+        return concurrentParts;
+    }
+
+
+    private static boolean entityTypeInAlreadyFound(Set<EntityRecognitionEntity> alreadyFound, String type) {
+
+        return !alreadyFound.stream().map(EntityRecognitionEntity::getType).collect(Collectors.toSet()).contains(type);
+    }
+
+
+    private static boolean offsetBetweenPartsIsLowerThanMax(Boundary currentBoundary, EntityRecognitionEntity entity) {
+
+        return Math.abs(entity.getStartOffset() - currentBoundary.end()) < MAX_DISTANCE_BETWEEN_PARTS;
+    }
+
+
+    private boolean nerEntityOffsetMatches(EntityRecognitionEntity nerEntity, TextBlock textBlock) {
+
+        return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset()));
+    }
+
+
+    private static void addSectionStartOffsetToEachNerEntity(List<Integer> stringOffsetsForMainSections, NerEntities nerEntities) {
+
+        nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {
+            int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key);
+            entityRecognitionEntity.setStartOffset(newStartOffset);
+            entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
+        }));
+    }
+
+
+    private static List<Integer> getStringStartOffsetsForMainSections(DocumentGraph documentGraph) {
+
+        return documentGraph.getMainSections().stream().map(SectionNode::buildTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -22,6 +22,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
@@ -32,19 +33,19 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
 
 import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class DroolsExecutionService {
 
-    private final RulesClient rulesClient;
+    RulesClient rulesClient;
-
+    Map<String, KieContainer> kieContainers = new HashMap<>();
-    private final Map<String, KieContainer> kieContainers = new HashMap<>();
+    Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
-
+    EntityCreationService entityCreationService;
-    private final Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
-
-    private final EntityCreationService entityCreationService;
 
 
     public KieContainer getKieContainer(String dossierTemplateId) {
@@ -63,35 +64,21 @@ public class DroolsExecutionService {
                                             DocumentGraph document,
                                             Dictionary dictionary,
                                             List<FileAttribute> fileAttributes,
-                                            ManualRedactions resizeRedactions) {
+                                            ManualRedactions manualRedactions,
+                                            List<EntityRecognitionEntity> nerEntities) {
 
-        KieSession kieSession = kieContainer.newKieSession();
+        return executeRules(kieContainer, document, document.getMainSections(), dictionary, fileAttributes, manualRedactions, nerEntities);
-        kieSession.setGlobal("document", document);
-        kieSession.setGlobal("entityCreationService", entityCreationService);
-        kieSession.setGlobal("dictionary", dictionary);
-
-        document.getEntities().forEach(kieSession::insert);
-        document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode()));
-        document.getPages().forEach(kieSession::insert);
-        fileAttributes.forEach(kieSession::insert);
-        if (resizeRedactions != null) {
-            resizeRedactions.getResizeRedactions().forEach(kieSession::insert);
-        }
-
-        kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
-        kieSession.fireAllRules();
-
-        return getFileAttributes(kieSession);
     }
 
 
     @Timed("redactmanager_executeRules")
     public List<FileAttribute> executeRules(KieContainer kieContainer,
                                             DocumentGraph document,
-                                            List<SectionNode> sectionsToReanalyze,
+                                            List<SectionNode> sectionsToAnalyze,
                                             Dictionary dictionary,
                                             List<FileAttribute> fileAttributes,
-                                            ManualRedactions resizeRedactions) {
+                                            ManualRedactions manualRedactions,
+                                            List<EntityRecognitionEntity> nerEntities) {
 
         KieSession kieSession = kieContainer.newKieSession();
         kieSession.setGlobal("document", document);
@@ -99,14 +86,21 @@ public class DroolsExecutionService {
         kieSession.setGlobal("dictionary", dictionary);
 
         document.getEntities().forEach(kieSession::insert);
-        sectionsToReanalyze.forEach(kieSession::insert);
+        sectionsToAnalyze.forEach(kieSession::insert);
-        sectionsToReanalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
+        sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
         document.getPages().forEach(kieSession::insert);
         fileAttributes.forEach(kieSession::insert);
-        if (resizeRedactions != null) {
+
-            resizeRedactions.getResizeRedactions().forEach(kieSession::insert);
+        if (manualRedactions != null) {
+            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
+            manualRedactions.getForceRedactions().forEach(kieSession::insert);
+            manualRedactions.getIdsToRemove().forEach(kieSession::insert);
+            manualRedactions.getImageRecategorization().forEach(kieSession::insert);
+            manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
         }
 
+        nerEntities.forEach(kieSession::insert);
+
         kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
         kieSession.fireAllRules();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -5,26 +5,19 @@ import static com.iqser.red.service.redaction.v1.server.document.graph.factory.R
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 
-import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -36,12 +29,15 @@ public class RedactionLogCreatorService {
     private final DictionaryService dictionaryService;
 
 
-    public List<RedactionLogEntry> createRedactionLog(Set<EntityNode> entityNodes, String dossierTemplateId) {
+    public List<RedactionLogEntry> createRedactionLog(DocumentGraph documentGraph, String dossierTemplateId) {
 
         List<RedactionLogEntry> entries = new ArrayList<>();
         Set<String> processedIds = new HashSet<>();
-        entityNodes.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)));
+        documentGraph.getEntities().forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)));
-
+        documentGraph.streamAllSubNodes()
+                .filter(node -> node instanceof ImageNode)
+                .map(node -> (ImageNode) node)
+                .forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
         return entries;
     }
 
@@ -106,193 +102,26 @@ public class RedactionLogCreatorService {
     }
 
 
-    @Timed("redactmanager_createRedactionLog")
+    public RedactionLogEntry createRedactionLogEntry(ImageNode image, String dossierTemplateId) {
-    public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages, String dossierTemplateId) {
 
-        List<RedactionLogEntry> entries = new ArrayList<>();
+        return RedactionLogEntry.builder()
-
+                .id(image.getId())
-        for (int page = 1; page <= numberOfPages; page++) {
+                .color(getColor(image.getImageType().toString(), dossierTemplateId, image.isRedaction()))
-            if (pageEntities.getEntitiesPerPage().get(page) != null) {
-                entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId));
-            }
-
-            if (pageEntities.getImagesPerPage().get(page) != null) {
-                entries.addAll(addImageEntries(pageEntities.getImagesPerPage(), page, dossierTemplateId));
-            }
-        }
-
-        return entries;
-    }
-
-
-    public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, int pageNumber, String dossierTemplateId) {
-
-        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
-
-        for (Image image : images.get(pageNumber)) {
-
-            String id = IdBuilder.buildId(image.getPosition(), pageNumber);
-
-            RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
-                    .id(id)
-                    .color(getColor(image.getType(), dossierTemplateId, image.isRedaction()))
                 .isImage(true)
-                    .type(image.getType())
+                .type(image.getImageType().toString())
                 .redacted(image.isRedaction())
                 .reason(image.getRedactionReason())
                 .legalBasis(image.getLegalBasis())
                 .matchedRule(image.getMatchedRule())
-                    .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
+                .isHint(dictionaryService.isHint(image.getImageType().toString(), dossierTemplateId))
                 .isDictionaryEntry(false)
                 .isRecommendation(false)
-                    .positions(List.of(new Rectangle(new Point((float) image.getPosition().getX(), (float) image.getPosition().getY()),
+                .positions(List.of(RectangleTransformations.toRectangle(image.getPosition(), image.getPage().getNumber())))
-                            (float) image.getPosition().getWidth(),
+                .sectionNumber(image.getTocId().get(0))
-                            (float) image.getPosition().getHeight(),
+                .section(image.getParent().buildTextBlock().getSearchText())
-                            pageNumber)))
+                .imageHasTransparency(image.isTransparent())
-                    .sectionNumber(image.getSectionNumber())
-                    .section(image.getSection())
-                    .imageHasTransparency(image.isHasTransparency())
                 .build();
 
-            redactionLogEntities.add(redactionLogEntry);
-        }
-
-        return redactionLogEntities;
-    }
-
-
-    public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId) {
-
-        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
-
-        // Duplicates can exist due table extraction columns over multiple rows.
-        Set<String> processedIds = new HashSet<>();
-
-        entityLoop:
-        for (Entity entity : entities.get(page)) {
-
-            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
-
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
-                if (processedIds.contains(entityPositionSequence.getId())) {
-
-                    // TODO refactor this outer loop jump as soon as we have the time.
-                    continue entityLoop;
-                } else {
-                    processedIds.add(entityPositionSequence.getId());
-                }
-
-                redactionLogEntry.setId(entityPositionSequence.getId());
-
-                if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
-                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences());
-
-                    redactionLogEntry.getPositions().addAll(rectanglesPerLine);
-
-                }
-
-                // FIXME ids should never be null. Figure out why this happens.
-                if (redactionLogEntry.getId() != null) {
-                    redactionLogEntities.add(redactionLogEntry);
-                }
-            }
-        }
-
-        return redactionLogEntities;
-    }
-
-
-    private List<Rectangle> getRectanglesPerLine(List<TextPositionSequence> textPositionSequences) {
-
-        List<Rectangle> rectangles = new ArrayList<>();
-
-        if (textPositionSequences.size() == 1) {
-            rectangles.add(textPositionSequences.get(0).getRectangle());
-            return rectangles;
-        }
-
-        TextPositionSequence combinedSequence = new TextPositionSequence();
-        for (int i = 0; i < textPositionSequences.size(); i++) {
-
-            if (combinedSequence.getTextPositions().isEmpty()) {
-                combinedSequence = textPositionSequences.get(i);
-                continue;
-            }
-
-            float lastSeqX = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getXDirAdj();
-            float lastSeqY = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getYDirAdj();
-            float lastSeqWidth = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getWidthDirAdj();
-            float lastSeqHeight = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getHeightDir();
-
-            float currentSeqX = textPositionSequences.get(i).getTextPositions().get(0).getXDirAdj();
-            float currentSeqY = textPositionSequences.get(i).getTextPositions().get(0).getYDirAdj();
-            float currentSeqHeight = textPositionSequences.get(i).getTextPositions().get(0).getHeightDir();
-
-            if (isCharInSameLine(lastSeqY, currentSeqY, lastSeqHeight, currentSeqHeight) && isCharClose(lastSeqX, currentSeqX, lastSeqWidth)) {
-                combinedSequence.getTextPositions().addAll(textPositionSequences.get(i).getTextPositions());
-            } else {
-                rectangles.add(combinedSequence.getRectangle());
-                combinedSequence = textPositionSequences.get(i);
-            }
-
-        }
-
-        if (!combinedSequence.getTextPositions().isEmpty()) {
-            rectangles.add(combinedSequence.getRectangle());
-        }
-
-        return rectangles;
-    }
-
-
-    private boolean isCharClose(float x, float xDirAdj, float width) {
-
-        float max = x + (5 * width);
-        return xDirAdj < max;
-    }
-
-
-    private boolean isCharInSameLine(float y, float yCompare, float height, float heightCompare) {
-
-        float offsetHeight = height / 2;
-        float minHeight = height - offsetHeight;
-        float maxHeight = height + offsetHeight;
-
-        float offsetY = height / 10;
-        float minY = y - offsetY;
-        float maxY = y + offsetY;
-
-        return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight;
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
-
-        Set<String> referenceIds = new HashSet<>();
-        entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId())));
-
-        return RedactionLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
-                .reason(entity.getRedactionReason())
-                .legalBasis(entity.getLegalBasis())
-                .value(entity.getWord())
-                .type(entity.getType())
-                .redacted(entity.isRedaction())
-                .isHint(isHint(entity.getType(), dossierTemplateId))
-                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(entity.getHeadline())
-                .sectionNumber(entity.getSectionNumber())
-                .matchedRule(entity.getMatchedRule())
-                .isDictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .startOffset(entity.getStart())
-                .endOffset(entity.getEnd())
-                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines())
-                .reference(referenceIds)
-                .build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -1,10 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Section;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -12,20 +21,22 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
 import lombok.RequiredArgsConstructor;
 
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.List;
-
 @Service
 @RequiredArgsConstructor
 public class SectionGridCreatorService {
 
-    public void createSectionGrid(Document classifiedDoc, int numberOfPages) {
+    public void createSectionGrid(Document classifiedDoc, int numberOfPages, List<SectionText> sectionTexts) {
 
         for (int page = 1; page <= numberOfPages; page++) {
             addSectionGrid(classifiedDoc, page);
         }
+        // enhance section grid with headline data
+        sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid()
+                .getSections()
+                .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
+                        sectionText.getHeadline(),
+                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
+                        sectionText.getSectionAreas())));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -2,11 +2,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -23,8 +21,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -32,13 +28,14 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified
 import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
@@ -72,6 +69,7 @@ public class AnalyzeService {
     private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
 
     DictionaryService dictionaryService;
+    NerEntitiesAdapter nerEntitiesAdapter;
     DroolsExecutionService droolsExecutionService;
     EntityRedactionService entityRedactionService;
     RedactionLogCreatorService redactionLogCreatorService;
@@ -118,21 +116,13 @@ public class AnalyzeService {
 
         List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
 
-        sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
+        sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount, sectionTexts);
 
         Text text = new Text(pageCount, sectionTexts);
 
-        // enhance section grid with headline data
-        sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid()
-                .getSections()
-                .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
-                        sectionText.getHeadline(),
-                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
-                        sectionText.getSectionAreas())));
-
         log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(documentGraph));
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(documentGraph));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 
         return AnalyzeResult.builder()
@@ -179,9 +169,7 @@ public class AnalyzeService {
 
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
 
-        NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
+        List<EntityRecognitionEntity> nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph);
-                ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
-                : new NerEntities();
 
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
 
@@ -192,7 +180,7 @@ public class AnalyzeService {
                 analyzeRequest,
                 nerEntities);
 
-        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId());
+        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -213,12 +201,7 @@ public class AnalyzeService {
         long startTime = System.currentTimeMillis();
         DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 
-        NerEntities nerEntities;
+        List<EntityRecognitionEntity> nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph);
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        } else {
-            nerEntities = new NerEntities();
-        }
         dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
         long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
@@ -231,13 +214,13 @@ public class AnalyzeService {
                 analyzeRequest,
                 nerEntities);
 
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId());
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
                 analyzeRequest.getAnalysisNumber(),
                 redactionLogEntries,
-                convert(legalBasis),
+                toSimplifiedSectionText(legalBasis),
                 dictionary.getVersion().getDossierTemplateVersion(),
                 dictionary.getVersion().getDossierVersion(),
                 rulesVersion,
@@ -296,7 +279,7 @@ public class AnalyzeService {
     }
 
 
-    public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {
+    public List<RedactionLogLegalBasis> toSimplifiedSectionText(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
     }
@@ -314,16 +297,16 @@ public class AnalyzeService {
     }
 
 
-    private SimplifiedText convert(Text text) {
+    private SimplifiedText toSimplifiedText(DocumentGraph documentGraph) {
 
-        List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+        List<SimplifiedSectionText> simplifiedSectionTexts = documentGraph.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
-        int numberOfPages = 0;
+        return SimplifiedText.builder().numberOfPages(documentGraph.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
-
-        if (text != null) {
-            numberOfPages = text.getNumberOfPages();
-            text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
     }
-        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
+
+
+    private SimplifiedSectionText toSimplifiedSectionText(SectionNode sectionNode) {
+
+        return SimplifiedSectionText.builder().sectionNumber(sectionNode.getTocId().get(0)).text(sectionNode.buildTextBlock().getSearchText()).build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -9,7 +9,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
@@ -37,13 +37,14 @@ public class EntityRedactionService {
                                                                                  DocumentGraph documentGraph,
                                                                                  KieContainer kieContainer,
                                                                                  AnalyzeRequest analyzeRequest,
-                                                                                 NerEntities nerEntities) {
+                                                                                 List<EntityRecognitionEntity> nerEntities) {
 
         List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,
                 documentGraph,
                 dictionary,
                 analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions());
+                analyzeRequest.getManualRedactions(),
+                nerEntities);
         return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
     }
 
@@ -53,14 +54,15 @@ public class EntityRedactionService {
                                                                                  List<SectionNode> sectionsToReanalyze,
                                                                                  KieContainer kieContainer,
                                                                                  AnalyzeRequest analyzeRequest,
-                                                                                 NerEntities nerEntities) {
+                                                                                 List<EntityRecognitionEntity> nerEntities) {
 
         List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,
                 documentGraph,
                 sectionsToReanalyze,
                 dictionary,
                 analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions());
+                analyzeRequest.getManualRedactions(),
+                nerEntities);
         return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -108,7 +108,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         kieSession.fireAllRules();
         System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - dictionaryAddsStart);
         System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document.getEntities(), TEST_DOSSIER_TEMPLATE_ID);
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID);
         drawAllEntities(filename, fileResource, document);
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 
 global DocumentGraph document
 global EntityCreationService entityCreationService
@@ -162,6 +163,16 @@ rule "run local dictionary search"
         entityNodes.forEach(entityNode -> insert(entityNode));
     end
 
+// --------------------------------------- NER Entities rules -------------------------------------------------------------------
+
+rule "add NER Entities of type CBI_author or CBI_address"
+    when
+        $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
+    then
+        EntityNode entityNode = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
+        insert(entityNode);
+    end
+
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
 rule "1: Redact CBI Authors (Non vertebrate study)"