RED-6093: Prototype implementation for document structure and analysis concept

*added a new analysis test to ensure the same entities are being found
Pull request #517 : RED-6264
2023-03-07 18:18:50 +01:00 · 2023-03-03 09:47:51 +01:00 · 2023-03-02 14:42:14 +01:00 · 2023-03-01 16:39:22 +01:00 · 2023-02-24 17:29:43 +01:00 · 2023-02-24 17:19:52 +01:00
53 changed files with 1647 additions and 670 deletions
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.13.0</version>
+        <version>1.17.0</version>
        <relativePath/>
    </parent>
    <modelVersion>4.0.0</modelVersion>
@@ -1,9 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;

-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -13,6 +9,10 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;

@@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;

-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -7,7 +7,7 @@ import lombok.experimental.UtilityClass;

@UtilityClass
@SuppressWarnings("all")
-public class PositionUtils {
+public final class PositionUtils {

    // TODO This currently uses pdf coord system. In the futher this should use java coord system.
    // Note: DirAdj (TextDirection Adjusted) can not be user for this.
@@ -8,7 +8,7 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import lombok.experimental.UtilityClass;

@UtilityClass
-public class RulingTextDirAdjustUtil {
+public final class RulingTextDirAdjustUtil {

    /**
     * Converts a ruling (line of a table) the same way TextPositions are converted in PDFBox.
@@ -25,16 +25,13 @@ public class RulingTextDirAdjustUtil {
    private Point2D convertPoint(float x, float y, float dir, float pageWidth, float pageHeight) {

        var xAdj = getXRot(x, y, dir, pageWidth, pageHeight);
-        var yAdj = 0f;
-        if (dir == 0 || dir == 180) {
-            yAdj = pageHeight - getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
-        } else {
-            yAdj = pageWidth - getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
-        }
+        var yLowerLeftRot = getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
+        var yAdj = dir == 0 || dir == 180 ? pageHeight - yLowerLeftRot : pageWidth - yLowerLeftRot;
        return new Point2D.Float(xAdj, yAdj);
    }


+    @SuppressWarnings("SuspiciousNameCombination")
    private float getXRot(float x, float y, float dir, float pageWidth, float pageHeight) {

        if (dir == 0) {
@@ -22,6 +22,7 @@ import org.apache.pdfbox.text.TextPosition;
 import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;

@Slf4j
@@ -200,6 +201,8 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        int startIndex = 0;
        RedTextPosition previous = null;

+        textPositions.sort(Comparator.comparing(TextPosition::getXDirAdj));
+
        for (int i = 0; i <= textPositions.size() - 1; i++) {

            if (!textPositionSequences.isEmpty()) {
@@ -14,8 +14,8 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;

 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -5,6 +5,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import lombok.Data;
 import lombok.Getter;

@@ -49,8 +50,9 @@ public class Dictionary {


    public DictionaryModel getType(String type) {
-
-        return localAccessMap.get(type);
+        DictionaryModel model = localAccessMap.get(type);
+        if (model == null) throw new NotFoundException("Type: " + type + " is not found");
+        return model;
    }


@@ -91,8 +91,8 @@ public class SearchableText {

                if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i).charAt(j, caseInsensitive) == '-') {

-                    if (counter != 0 || i == 0 && j == 0 || j != 0 && SeparatorUtils.isSeparator(searchSpace.get(i)
-                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1)
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && (SeparatorUtils.isSeparator(searchSpace.get(i).charAt(j - 1, caseInsensitive)) || SeparatorUtils.isSeparator(
+                            searchSpace.get(i).charAt(j, caseInsensitive))) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i).charAt(j, caseInsensitive) != ' ') {
                        partMatch.add(searchSpace.get(i), searchSpace.get(i).textPositionAt(j));
@@ -105,9 +105,10 @@ public class SearchableText {
                        crossSequenceParts.add(partMatch);

                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
-                                .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i).charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
-                                .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1).charAt(0, caseInsensitive)) || j == searchSpace.get(i)
-                                .length() - 1 && searchSpace.get(i).charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1).charAt(0, caseInsensitive) != ' ') {
+                                .length() - 1 && (SeparatorUtils.isSeparator(searchSpace.get(i).charAt(j + 1, caseInsensitive)) || SeparatorUtils.isSeparator(searchSpace.get(i)
+                                .charAt(j, caseInsensitive))) || j == searchSpace.get(i).length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1)
+                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1).charAt(0, caseInsensitive) != ' ') {
                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
                        }

@@ -118,7 +119,7 @@ public class SearchableText {
                } else {
                    counter = 0;
                    if (!crossSequenceParts.isEmpty()) {
-                        j--;
+                        j = j - partMatch.length() - 1;
                    }
                    crossSequenceParts = new ArrayList<>();
                    partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
@@ -37,7 +37,7 @@ public class ManualRedactionSurroundingTextService {
    private final SurroundingWordsService surroundingWordsService;


-    @Timed("redactmanager_SurroundingTextAnalysis")
+    @Timed("redactmanager_surroundingTextAnalysis")
    public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {

        long startTime = System.currentTimeMillis();
@@ -257,11 +257,11 @@ public class RedactionLogMergeService {
                    if (manualOverrideReason != null) {
                        redactionLogEntry.setReason(manualOverrideReason);
                    }
-
-                    redactionLogEntry.getManualChanges()
-                            .add(ManualChange.from(manualRemoval)
-                                    .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
                }
+
+                redactionLogEntry.getManualChanges()
+                        .add(ManualChange.from(manualRemoval)
+                                .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
            }

            if (mrw.getItem() instanceof ManualForceRedaction) {
@@ -334,8 +334,9 @@ public class RedactionLogMergeService {
                if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-                    redactionLogEntry.setValue(manualResizeRedact.getValue());
-
+                    if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
+                        redactionLogEntry.setValue(manualResizeRedact.getValue());
+                    }
                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
@@ -43,7 +43,7 @@ public class SurroundingWordsService {
    }


-    @Timed("redactmanager_addSurroundingText_tables")
+    @Timed("redactmanager_addSurroundingTextTables")
    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {

        if (entities.isEmpty()) {
@@ -1,33 +1,25 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;

 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;

 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;

 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
@@ -44,42 +36,94 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionChangeLogService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SectionGridCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SectionTextBuilderService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction.EntityRedactionService;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

 import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
+import io.micrometer.core.instrument.FunctionTimer;
+import io.micrometer.core.instrument.MeterRegistry;
+import lombok.AccessLevel;
+import lombok.Getter;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
-@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class AnalyzeService {

-    private final DictionaryService dictionaryService;
-    private final DroolsExecutionService droolsExecutionService;
-    private final EntityRedactionService entityRedactionService;
-    private final RedactionLogCreatorService redactionLogCreatorService;
-    private final RedactionStorageService redactionStorageService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionChangeLogService redactionChangeLogService;
-    private final LegalBasisClient legalBasisClient;
-    private final RedactionServiceSettings redactionServiceSettings;
-    private final SectionTextBuilderService sectionTextBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
-    private final ImageService imageService;
-    private final ImportedRedactionService importedRedactionService;
+    DictionaryService dictionaryService;
+    DroolsExecutionService droolsExecutionService;
+    EntityRedactionService entityRedactionService;
+    RedactionLogCreatorService redactionLogCreatorService;
+    RedactionStorageService redactionStorageService;
+    PdfSegmentationService pdfSegmentationService;
+    RedactionChangeLogService redactionChangeLogService;
+    LegalBasisClient legalBasisClient;
+    RedactionServiceSettings redactionServiceSettings;
+    SectionTextBuilderService sectionTextBuilderService;
+    SectionGridCreatorService sectionGridCreatorService;
+    ImageService imageService;
+    ImportedRedactionService importedRedactionService;
+    SectionFinder sectionFinder;
+    PagewiseCounter analyzeCounter = new PagewiseCounter();
+
+
+    @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
+    public AnalyzeService(DictionaryService dictionaryService,
+                          DroolsExecutionService droolsExecutionService,
+                          EntityRedactionService entityRedactionService,
+                          RedactionLogCreatorService redactionLogCreatorService,
+                          RedactionStorageService redactionStorageService,
+                          PdfSegmentationService pdfSegmentationService,
+                          RedactionChangeLogService redactionChangeLogService,
+                          LegalBasisClient legalBasisClient,
+                          RedactionServiceSettings redactionServiceSettings,
+                          SectionTextBuilderService sectionTextBuilderService,
+                          SectionGridCreatorService sectionGridCreatorService,
+                          ImageService imageService,
+                          ImportedRedactionService importedRedactionService,
+                          SectionFinder sectionFinder,
+                          MeterRegistry meterRegistry) {
+
+        this.dictionaryService = dictionaryService;
+        this.droolsExecutionService = droolsExecutionService;
+        this.entityRedactionService = entityRedactionService;
+        this.redactionLogCreatorService = redactionLogCreatorService;
+        this.redactionStorageService = redactionStorageService;
+        this.pdfSegmentationService = pdfSegmentationService;
+        this.redactionChangeLogService = redactionChangeLogService;
+        this.legalBasisClient = legalBasisClient;
+        this.redactionServiceSettings = redactionServiceSettings;
+        this.sectionTextBuilderService = sectionTextBuilderService;
+        this.sectionGridCreatorService = sectionGridCreatorService;
+        this.imageService = imageService;
+        this.importedRedactionService = importedRedactionService;
+        this.sectionFinder = sectionFinder;
+
+        // This is just here as an example of how to do a timer with a counter.
+        // For a full implementation this should be moved to a common helper class.
+        FunctionTimer.builder("redactmanager_analyze.pagewise", //
+                        analyzeCounter, //
+                        PagewiseCounter::getTotalPageCount, //
+                        PagewiseCounter::getTotalTimeMillis, //
+                        TimeUnit.MILLISECONDS) //
+                .register(meterRegistry);
+    }


    @Timed("redactmanager_analyzeDocumentStructure")
@@ -153,32 +197,25 @@ public class AnalyzeService {
                new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
                analyzeRequest.getDossierId());

-        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(
-                dictionaryIncrement,
-                redactionLog,
-                text,
-                analyzeRequest);
+        Set<Integer> sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() //
+                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) //
+                : analyzeRequest.getSectionsToReanalyse();
+        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);

        if (sectionsToReanalyse.isEmpty()) {
            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
        }

-        NerEntities nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        } else {
-            nerEntities = new NerEntities();
-        }
-
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
        List<SectionText> reanalysisSections = text.getSectionTexts()
                .stream()
                .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                .collect(Collectors.toList());
-
+        NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
+                ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
+                : new NerEntities();
        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);

        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
@@ -238,48 +275,13 @@ public class AnalyzeService {
    }


-    @Timed("redactmanager_findSectionsToReanalyse")
-    private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
-
-        long start = System.currentTimeMillis();
-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
-
-        Set<Integer> sectionsToReanalyse = new HashSet<>();
-        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
-        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                sectionsToReanalyse.add(entry.getSectionNumber());
-            }
-            if (entry.isImage()) {
-                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
-            }
-        }
-
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
-
-        for (SectionText sectionText : text.getSectionTexts()) {
-
-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
-                sectionsToReanalyse.add(sectionText.getSectionNumber());
-            }
-
-        }
-
-        log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);
-
-        return sectionsToReanalyse;
-    }
-
-
    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                           long startTime,
                                           RedactionLog redactionLog,
                                           Text text,
                                           DictionaryVersion dictionaryVersion,
                                           boolean isReanalysis,
-                                           Set<FileAttribute> addedFileAttributes
-    ) {
+                                           Set<FileAttribute> addedFileAttributes) {

        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
@@ -294,6 +296,8 @@ public class AnalyzeService {

        long duration = System.currentTimeMillis() - startTime;

+        analyzeCounter.increase(text.getNumberOfPages(), duration);
+
        return AnalyzeResult.builder()
                .dossierId(analyzeRequest.getDossierId())
                .fileId(analyzeRequest.getFileId())
@@ -313,41 +317,12 @@ public class AnalyzeService {
    }


-    private Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
-
-        if (manualRedactions == null) {
-            return new HashSet<>();
-        }
-
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
-    }
-
-
    public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {

        return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
    }


-    public Image convert(RedactionLogEntry entry) {
-
-        Rectangle position = entry.getPositions().get(0);
-
-        return Image.builder()
-                .type(entry.getType())
-                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
-                .sectionNumber(entry.getSectionNumber())
-                .section(entry.getSection())
-                .page(position.getPage())
-                .hasTransparency(entry.isImageHasTransparency())
-                .build();
-    }
-
-
    private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {

        if (excludedPages != null && !excludedPages.isEmpty()) {
@@ -372,4 +347,21 @@ public class AnalyzeService {
        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
    }

+
+    @FieldDefaults(level = AccessLevel.PRIVATE)
+    @Getter
+    private static final class PagewiseCounter {
+
+        long totalPageCount;
+        long totalTimeMillis;
+
+
+        public void increase(long pageCount, long durationMillis) {
+
+            totalPageCount += pageCount;
+            totalTimeMillis += durationMillis;
+        }
+
+    }
+
 }
@@ -0,0 +1,101 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.Text;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+class SectionFinder {
+    @Timed("redactmanager_findSectionsToReanalyse")
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
+
+        long start = System.currentTimeMillis();
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        Set<Integer> sectionsToReanalyse = new HashSet<>();
+        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
+        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
+            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+                sectionsToReanalyse.add(entry.getSectionNumber());
+            }
+            if (entry.isImage()) {
+                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
+            }
+        }
+
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
+                true);
+
+        for (SectionText sectionText : text.getSectionTexts()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
+                sectionsToReanalyse.add(sectionText.getSectionNumber());
+            }
+
+        }
+
+        log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
+
+        return sectionsToReanalyse;
+    }
+
+    private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null) {
+            return new HashSet<>();
+        }
+
+        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
+                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
+                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
+                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
+                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+    }
+
+    private static Image convert(RedactionLogEntry entry) {
+
+        Rectangle position = entry.getPositions().get(0);
+
+        return Image.builder()
+                .type(entry.getType())
+                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
+                .sectionNumber(entry.getSectionNumber())
+                .section(entry.getSection())
+                .page(position.getPage())
+                .hasTransparency(entry.isImageHasTransparency())
+                .build();
+    }
+}
@@ -0,0 +1,127 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+class EntityFinder {
+
+    RedactionServiceSettings redactionServiceSettings;
+
+
+    @Timed("redactmanager_findEntities")
+    public Entities findEntities(SearchableText searchableText,
+                                 String headline,
+                                 int sectionNumber,
+                                 Dictionary dictionary,
+                                 boolean local,
+                                 NerEntities nerEntities,
+                                 List<Integer> cellStarts,
+                                 ManualRedactions manualRedactions) {
+
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
+
+        if (StringUtils.isEmpty(searchableString)) {
+            return new Entities(new HashSet<>(), new HashSet<>());
+        }
+
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+
+            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !local,
+                            model.isDossierDictionary(),
+                            local ? Engine.RULE : Engine.DICTIONARY,
+                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
+        }
+
+        Set<Entity> nerFound = new HashSet<>();
+        if (!local) {
+            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
+        }
+
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
+    }
+
+
+    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
+
+        Set<Entity> entities = new HashSet<>();
+
+        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
+            nerEntities.getData().get(sectionNumber).forEach(res -> {
+                if (cellStarts == null || cellStarts.isEmpty()) {
+                    entities.add(new Entity(res.getValue(),
+                            res.getType(),
+                            res.getStartOffset(),
+                            res.getEndOffset(),
+                            headline,
+                            sectionNumber,
+                            false,
+                            false,
+                            Engine.NER,
+                            EntityType.RECOMMENDATION));
+                } else {
+                    boolean intersectsCellStart = false;
+                    for (Integer cellStart : cellStarts) {
+                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
+                            intersectsCellStart = true;
+                            break;
+                        }
+                    }
+                    if (!intersectsCellStart) {
+                        entities.add(new Entity(res.getValue(),
+                                res.getType(),
+                                res.getStartOffset(),
+                                res.getEndOffset(),
+                                headline,
+                                sectionNumber,
+                                false,
+                                false,
+                                Engine.NER,
+                                EntityType.RECOMMENDATION));
+                    }
+                }
+            });
+        }
+        return entities;
+    }
+
+}
@@ -1,41 +1,52 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

-import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
@Slf4j
@Service
@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityRedactionService {

-    private final RedactionServiceSettings redactionServiceSettings;
-    private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;
+    DroolsExecutionService droolsExecutionService;
+    SurroundingWordsService surroundingWordsService;
+    EntityFinder entityFinder;


    public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -45,7 +56,7 @@ public class EntityRedactionService {

        if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {

-            if(!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
+            if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
                //AnalyzeRequest provides immutable list.
                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
                mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
@@ -54,7 +65,14 @@ public class EntityRedactionService {
            }

            Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
-            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
+            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
+                    dictionary,
+                    kieContainer,
+                    analyzeRequest,
+                    true,
+                    hintsPerSectionNumber,
+                    imagesPerPage,
+                    nerEntities);
            EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
            EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
        }
@@ -67,18 +85,78 @@ public class EntityRedactionService {


    public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
-                                    Dictionary dictionary,
-                                    KieContainer kieContainer,
-                                    AnalyzeRequest analyzeRequest,
-                                    boolean local,
-                                    Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                    Map<Integer, Set<Image>> imagesPerPage,
-                                    NerEntities nerEntities) {
+                                           Dictionary dictionary,
+                                           KieContainer kieContainer,
+                                           AnalyzeRequest analyzeRequest,
+                                           boolean local,
+                                           Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                           Map<Integer, Set<Image>> imagesPerPage,
+                                           NerEntities nerEntities) {

-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
-        for (SectionText reanalysisSection : reanalysisSections) {
+        List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
+                dictionary,
+                analyzeRequest,
+                local,
+                hintsPerSectionNumber,
+                nerEntities);

-            Entities entities = findEntities(reanalysisSection.getSearchableText(),
+        Set<FileAttribute> addedFileAttributes = new HashSet<>();
+        Set<Entity> entities = new HashSet<>();
+        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+
+            if (!addedFileAttributes.isEmpty()) {
+                //Section.Builder provides immutable list.
+                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
+                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
+                mergedFileAttributes.addAll(addedFileAttributes);
+                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
+            }
+
+            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+
+            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
+
+            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+
+            var entriesWithoutSurroundingText = analysedSection.getEntities()
+                    .stream()
+                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
+                    .collect(Collectors.toSet());
+
+            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                        sectionSearchableTextPair.getSearchableText(),
+                        dictionary,
+                        sectionSearchableTextPair.getCellStarts());
+            } else {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+            }
+
+            entities.addAll(analysedSection.getEntities());
+
+            if (!local) {
+                for (Image image : analysedSection.getImages()) {
+                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+                }
+                addLocalValuesToDictionary(analysedSection, dictionary);
+            }
+
+        });
+
+        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
+    }
+
+
+    private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
+                                                                       Dictionary dictionary,
+                                                                       AnalyzeRequest analyzeRequest,
+                                                                       boolean local,
+                                                                       Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                                       NerEntities nerEntities) {
+
+        return reanalysisSections.stream().map(reanalysisSection -> {
+
+            Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
                    reanalysisSection.getHeadline(),
                    reanalysisSection.getSectionNumber(),
                    dictionary,
@@ -136,73 +214,36 @@ public class EntityRedactionService {

            log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());

-            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
-                    .isLocal(false)
-                    .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
-                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                    .nerEntities(entities.getNerEntities())
-                    .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
-                    .searchText(reanalysisSection.getSearchableText().toString())
-                    .headline(reanalysisSection.getHeadline())
-                    .sectionNumber(reanalysisSection.getSectionNumber())
-                    .tabularData(reanalysisSection.getTabularData())
-                    .searchableText(reanalysisSection.getSearchableText())
-                    .dictionary(dictionary)
-                    .images(reanalysisSection.getImages())
-                    .sectionAreas(reanalysisSection.getSectionAreas())
-                    .fileAttributes(analyzeRequest.getFileAttributes())
-                    .manualRedactions(analyzeRequest.getManualRedactions())
-                    .isInTable(reanalysisSection.isTable())
-                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
-
-        }
+            return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
+        }).collect(Collectors.toList());
+    }


-        Set<FileAttribute> addedFileAttributes = new HashSet<>();
-        Set<Entity> entities = new HashSet<>();
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+    private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary,
+                                                                  AnalyzeRequest analyzeRequest,
+                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                                  SectionText reanalysisSection,
+                                                                  Entities entities) {

-            if(!addedFileAttributes.isEmpty()) {
-                //Section.Builder provides immutable list.
-                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
-                mergedFileAttributes.addAll(addedFileAttributes);
-                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
-            }
-
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
-
-            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
-
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
-
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
-                    .stream()
-                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
-                    .collect(Collectors.toSet());
-
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
-                        dictionary,
-                        sectionSearchableTextPair.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
-            }
-
-            entities.addAll(analysedSection.getEntities());
-
-            if (!local) {
-                for (Image image : analysedSection.getImages()) {
-                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
-                }
-                addLocalValuesToDictionary(analysedSection, dictionary);
-            }
-
-        });
-
-        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
+        return new SectionSearchableTextPair(Section.builder()
+                .isLocal(false)
+                .dictionaryTypes(dictionary.getTypes())
+                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(),
+                        hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
+                .nerEntities(entities.getNerEntities())
+                .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
+                .searchText(reanalysisSection.getSearchableText().toString())
+                .headline(reanalysisSection.getHeadline())
+                .sectionNumber(reanalysisSection.getSectionNumber())
+                .tabularData(reanalysisSection.getTabularData())
+                .searchableText(reanalysisSection.getSearchableText())
+                .dictionary(dictionary)
+                .images(reanalysisSection.getImages())
+                .sectionAreas(reanalysisSection.getSectionAreas())
+                .fileAttributes(analyzeRequest.getFileAttributes())
+                .manualRedactions(analyzeRequest.getManualRedactions())
+                .isInTable(reanalysisSection.isTable())
+                .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts());
    }


@@ -244,7 +285,7 @@ public class EntityRedactionService {
    private Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {

        Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
-        entities.stream().forEach(entity -> {
+        entities.forEach(entity -> {
            if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
            }
@@ -269,93 +310,4 @@ public class EntityRedactionService {
        }));
    }

-
-    @Timed("redactmanager_findEntities")
-    private Entities findEntities(SearchableText searchableText,
-                                  String headline,
-                                  int sectionNumber,
-                                  Dictionary dictionary,
-                                  boolean local,
-                                  NerEntities nerEntities,
-                                  List<Integer> cellStarts,
-                                  ManualRedactions manualRedactions) {
-
-        Set<Entity> found = new HashSet<>();
-        String searchableString = searchableText.asString();
-
-        if (StringUtils.isEmpty(searchableString)) {
-            return new Entities(new HashSet<>(), new HashSet<>());
-        }
-
-        String lowercaseInputString = searchableString.toLowerCase();
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-
-            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
-            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
-                    searchImplementation,
-                    model,
-                    new FindEntityDetails(model.getType(),
-                            headline,
-                            sectionNumber,
-                            !local,
-                            model.isDossierDictionary(),
-                            local ? Engine.RULE : Engine.DICTIONARY,
-                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
-
-            EntitySearchUtils.addOrAddEngine(found, entities);
-        }
-
-        Set<Entity> nerFound = new HashSet<>();
-        if (!local) {
-            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
-        }
-
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
-    }
-
-
-    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
-
-        Set<Entity> entities = new HashSet<>();
-
-        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
-            nerEntities.getData().get(sectionNumber).forEach(res -> {
-                if (cellStarts == null || cellStarts.isEmpty()) {
-                    entities.add(new Entity(res.getValue(),
-                            res.getType(),
-                            res.getStartOffset(),
-                            res.getEndOffset(),
-                            headline,
-                            sectionNumber,
-                            false,
-                            false,
-                            Engine.NER,
-                            EntityType.RECOMMENDATION));
-                } else {
-                    boolean intersectsCellStart = false;
-                    for (Integer cellStart : cellStarts) {
-                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
-                            intersectsCellStart = true;
-                            break;
-                        }
-                    }
-                    if (!intersectsCellStart) {
-                        entities.add(new Entity(res.getValue(),
-                                res.getType(),
-                                res.getStartOffset(),
-                                res.getEndOffset(),
-                                headline,
-                                sectionNumber,
-                                false,
-                                false,
-                                Engine.NER,
-                                EntityType.RECOMMENDATION));
-                    }
-                }
-            });
-        }
-        return entities;
-    }
-
 }
@@ -1,20 +1,33 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;

 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

-import java.util.*;
-import java.util.stream.Collectors;
-
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
-public class EntitySearchUtils {
+public final class EntitySearchUtils {

    public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {

@@ -26,7 +39,7 @@ public class EntitySearchUtils {

        Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
        markFalsePositives(found, falsePositives);
-        found.removeIf(f -> f.isFalsePositive());
+        found.removeIf(Entity::isFalsePositive);
    }


@@ -58,8 +71,8 @@ public class EntitySearchUtils {

    private void validateAndAddEntity(Set<Entity> entities, FindEntityDetails findEntityDetails, String inputString, int startIndex, int stopIndex) {

-        if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(inputString.charAt(
-                stopIndex)))) {
+        if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1)) || SeparatorUtils.isSeparator(inputString.charAt(startIndex))) && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(
+                inputString.charAt(stopIndex)) || SeparatorUtils.isSeparator(inputString.charAt(stopIndex - 1)))) {
            entities.add(new Entity(inputString.substring(startIndex, stopIndex),
                    findEntityDetails.getType(),
                    startIndex,
@@ -292,9 +305,12 @@ public class EntitySearchUtils {
                .get(0)
                .getSequences()
                .get(0)
-                .getMinXDirAdj() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxXDirAdj() && image.getPosition()
-                .getY() < entity.getPositionSequences().get(0).getSequences().get(0).getMinYDirAdj() && image.getPosition().getY() + image.getPosition()
-                .getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxYDirAdj();
+                .getMinXDirAdj() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences()
+                .get(0)
+                .getSequences()
+                .get(0)
+                .getMaxXDirAdj() && image.getPosition().getY() < entity.getPositionSequences().get(0).getSequences().get(0).getMinYDirAdj() && image.getPosition()
+                .getY() + image.getPosition().getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getMaxYDirAdj();
    }


@@ -1,5 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -7,11 +10,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;

 import lombok.experimental.UtilityClass;

-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
@UtilityClass
-public class IdBuilder {
+public final class IdBuilder {

    private final HashFunction hashFunction = Hashing.murmur3_128();

@@ -10,7 +10,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.OffsetString;
 import lombok.experimental.UtilityClass;

@UtilityClass
-public class OffsetStringUtils {
+public final class OffsetStringUtils {

    /**
     * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only.
@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import lombok.experimental.UtilityClass;
-
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Pattern;

+import lombok.experimental.UtilityClass;
+
@UtilityClass
-public class Patterns {
+public final class Patterns {

    public static Map<String, Pattern> patternCache = new HashMap<>();

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import lombok.experimental.UtilityClass;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -10,8 +8,10 @@ import java.nio.charset.StandardCharsets;
 import java.util.Set;
 import java.util.stream.Collectors;

+import lombok.experimental.UtilityClass;
+
@UtilityClass
-public class ResourceLoader {
+public final class ResourceLoader {

    public Set<String> load(String classpathPath) {

@@ -1,24 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import lombok.extern.slf4j.Slf4j;
-
 import java.util.Set;
 import java.util.regex.Pattern;

+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
-public class SeparatorUtils {
+@UtilityClass
+public final class SeparatorUtils {

    private final static Pattern punctuationPattern = Pattern.compile("\\p{Punct}");
-    private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E'
-            , '\u2039', '\u203A');
+    private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E', '\u2039', '\u203A');

    private final static Set<Integer> japaneseAltPunctuationMarks = Set.of(65288, 65289, 65294, 65339, 65341, 65371, 65373, 65375, 65376, 12443, 12444, 65309, 65306);

+
    public static boolean isSeparator(char c) {
+
        return Character.isWhitespace(c) || punctuationPattern.matcher(String.valueOf(c)).matches() || quotes.contains(c) || isJapaneseSeparator(c);
    }

+
    public static boolean isJapaneseSeparator(char c) {
+
        var intValue = (int) c;
        return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
    }
@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 import lombok.experimental.UtilityClass;

@UtilityClass
-public class TextNormalizationUtilities {
+public final class TextNormalizationUtilities {

    /**
     * Revert hyphenation due to line breaks.
@@ -4,18 +4,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;

 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SystemUtils;
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -36,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -58,25 +53,14 @@ public class PdfSegmentationService {
    public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException {

        PDDocument pdDocument = null;
+        File tempFile = null;
        try {
-            //create tempFile
-            File tempFile;
-
-            if (SystemUtils.IS_OS_UNIX) {
-                FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
-                tempFile = Files.createTempFile("document", ".pdf", attr).toFile();
-            } else {
-                tempFile = Files.createTempFile("document", ".pdf").toFile();
-                tempFile.setReadable(true, true);
-                tempFile.setWritable(true, true);
-                tempFile.setExecutable(true, true);
-            }
-
            Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>();
            if (redactionServiceSettings.isCvTableParsingEnabled()) {
                pdfTableCells = tableService.convertTables(dossierId, fileId);
            }

+            tempFile = FileUtils.createTempFile("document", ".pdf");
            try (var fos = new FileOutputStream(tempFile)) {
                IOUtils.copy(documentInputStream, fos);

@@ -89,43 +73,7 @@ public class PdfSegmentationService {
                long pageCount = pdDocument.getNumberOfPages();

                for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-
-                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                    stripper.setPageNumber(pageNumber);
-                    stripper.setStartPage(pageNumber);
-                    stripper.setEndPage(pageNumber);
-                    stripper.setPdpage(pdPage);
-                    stripper.getText(pdDocument);
-
-                    PDRectangle pdr = pdPage.getMediaBox();
-
-                    int rotation = pdPage.getRotation();
-                    boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-                    PDRectangle cropbox = pdPage.getCropBox();
-                    CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                            stripper.getRulings(),
-                            stripper.getMinCharWidth(),
-                            stripper.getMaxCharHeight());
-                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-                    page.setRotation(rotation);
-                    page.setLandscape(isLandscape);
-                    page.setPageNumber(pageNumber);
-                    page.setPageWidth(cropbox.getWidth());
-                    page.setPageHeight(cropbox.getHeight());
-
-                    tableExtractionService.extractTables(cleanRulings, page);
-                    buildPageStatistics(page);
-                    increaseDocumentStatistics(page, document);
-
-                    if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-                        page.setImages(pdfImages.get(pageNumber));
-                        imageService.findOcr(page);
-                    }
-
-                    pages.add(page);
+                    processPage(pdfImages, pdDocument, pdfTableCells, document, pages, pageNumber);
                }

                document.setPages(pages);
@@ -134,22 +82,65 @@ public class PdfSegmentationService {
                sectionsBuilderService.buildSections(document);
                sectionsBuilderService.addImagesToSections(document);

-                IOUtils.close(pdDocument);
-
-                if (!tempFile.delete()) {
-                    log.warn("Could not delete tmp file");
-                }
-
                return document;
            }
        } finally {
            if (pdDocument != null) {
                pdDocument.close();
            }
+
+            FileUtils.deleteFile(tempFile);
        }
    }


+    private void processPage(Map<Integer, List<PdfImage>> pdfImages,
+                           PDDocument pdDocument,
+                           Map<Integer, List<PdfTableCell>> pdfTableCells,
+                           Document document,
+                           List<Page> pages,
+                           int pageNumber) throws IOException {
+
+        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+        stripper.setPageNumber(pageNumber);
+        stripper.setStartPage(pageNumber);
+        stripper.setEndPage(pageNumber);
+        stripper.setPdpage(pdPage);
+        stripper.getText(pdDocument);
+
+        PDRectangle pdr = pdPage.getMediaBox();
+
+        int rotation = pdPage.getRotation();
+        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+        PDRectangle cropbox = pdPage.getCropBox();
+        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                stripper.getRulings(),
+                stripper.getMinCharWidth(),
+                stripper.getMaxCharHeight());
+        Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
+        page.setRotation(rotation);
+        page.setLandscape(isLandscape);
+        page.setPageNumber(pageNumber);
+        page.setPageWidth(cropbox.getWidth());
+        page.setPageHeight(cropbox.getHeight());
+
+        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+            page.setImages(pdfImages.get(pageNumber));
+            imageService.findOcr(page);
+        }
+
+        tableExtractionService.extractTables(cleanRulings, page);
+        buildPageStatistics(page);
+        increaseDocumentStatistics(page, document);
+
+        pages.add(page);
+    }
+
+
    private void increaseDocumentStatistics(Page page, Document document) {

        if (!page.isLandscape()) {
@@ -1,58 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.tableextraction.model;
-
-import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
-
-import org.locationtech.jts.geom.Envelope;
-import org.locationtech.jts.index.strtree.STRtree;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@SuppressWarnings("all")
-public class RectangleSpatialIndex<T extends Rectangle> {
-
-    private final STRtree si = new STRtree();
-    private final List<T> rectangles = new ArrayList<>();
-
-
-    public void add(T te) {
-
-        rectangles.add(te);
-        si.insert(new Envelope(te.getLeft(), te.getRight(), te.getBottom(), te.getTop()), te);
-    }
-
-
-    public List<T> contains(Rectangle rectangle) {
-
-        List<T> intersection = si.query(new Envelope(rectangle.getLeft(), rectangle.getRight(), rectangle.getTop(), rectangle.getBottom()));
-        List<T> rv = new ArrayList<T>();
-
-        for (T ir : intersection) {
-            if (rectangle.contains(ir)) {
-                rv.add(ir);
-            }
-        }
-
-        Utils.sort(rv, Rectangle.ILL_DEFINED_ORDER);
-        return rv;
-    }
-
-
-    public List<T> intersects(Rectangle r) {
-
-        List rv = si.query(new Envelope(r.getLeft(), r.getRight(), r.getTop(), r.getBottom()));
-        return rv;
-    }
-
-
-    /**
-     * Minimum bounding box of all the Rectangles contained on this RectangleSpatialIndex
-     *
-     * @return a Rectangle
-     */
-    public Rectangle getBounds() {
-
-        return Rectangle.boundingBoxOf(rectangles);
-    }
-
-}
@@ -1,18 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;

+import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;

 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;

 import lombok.Getter;
 import lombok.Setter;
@@ -23,7 +23,6 @@ public class Table extends AbstractTextContainer {

    private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();

-    private final RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
    private final int rotation;
    @Getter
    @Setter
@@ -208,80 +207,74 @@ public class Table extends AbstractTextContainer {
            return;
        }

-        Iterator<Cell> itty = cells.iterator();
+        cells.removeIf(cell -> cell.getWidth() < 1.1 || cell.getHeight() < 1.1);

-        while (itty.hasNext()) {
-            Cell cell = itty.next();
-            if (cell.getWidth() > 1.1 && cell.getHeight() > 1.1) {
-                si.add(cell);
-            } else {
-                itty.remove();
-            }
-        }
+        List<List<Cell>> rowsOfCells = calculateStructure(cells);

-        List<List<Cell>> rowsOfCells = rowsOfCells(cells);
-
-        Map<Integer, Cell> previousNonNullCellForColumnIndex = new HashMap<>();
        for (int i = 0; i < rowsOfCells.size(); i++) {
-            List<Cell> row = rowsOfCells.get(i);
-            Iterator<Cell> rowCells = row.iterator();
-            int startColumn = 0;
-            int jumpToColumn = 0;
-            while (rowCells.hasNext()) {
-                Cell cell = rowCells.next();
-                if (i > 0) {
-                    Rectangle rectangle = new Rectangle(cell.getBottom(),
-                            si.getBounds().getLeft(),
-                            cell.getLeft() - si.getBounds().getLeft() + 1,
-                            si.getBounds().getBottom() - cell.getBottom());
-                    List<List<Cell>> others = rowsOfCells(si.contains(rectangle));
-
-                    for (List<Cell> r : others) {
-                        jumpToColumn = Math.max(jumpToColumn, r.size());
-                    }
-
-                    while (startColumn != jumpToColumn) {
-                        add(previousNonNullCellForColumnIndex.get(startColumn), i, startColumn);
-                        startColumn++;
-                    }
-                }
-                add(cell, i, startColumn);
-                previousNonNullCellForColumnIndex.put(startColumn, cell);
-                startColumn++;
-                jumpToColumn = startColumn;
+            for (int j = 0; j < rowsOfCells.get(i).size(); j++) {
+                add(rowsOfCells.get(i).get(j), i, j);
            }
        }
+
    }


-    private List<List<Cell>> rowsOfCells(List<Cell> cells) {
+    /**
+     * Calculates the structure of the table. For spanning rows and columns multiple cells with the same values will be inserted.
+     *
+     * @param cells The found cells
+     * @return Table Structure
+     */
+    private List<List<Cell>> calculateStructure(List<Cell> cells) {

-        List<List<Cell>> rv = new ArrayList<>();
+        List<List<Cell>> matrix = new ArrayList<>();

        if (cells.isEmpty()) {
-            return rv;
+            return matrix;
        }
-        cells.sort(Comparator.comparingDouble(Rectangle::getLeft));

-        cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(), 2))));
+        Set<Float> uniqueX = new HashSet<>();
+        Set<Float> uniqueY = new HashSet<>();
+        cells.stream().filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3).forEach(c -> {
+            uniqueX.add(c.getLeft());
+            uniqueX.add(c.getRight());
+            uniqueY.add(c.getBottom());
+            uniqueY.add(c.getTop());
+        });

-        Iterator<Cell> iter = cells.iterator();
-        Cell c = iter.next();
-        float lastTop = c.getBottom();
-        List<Cell> lastRow = new ArrayList<>();
-        lastRow.add(c);
-        rv.add(lastRow);
+        var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList());
+        var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList());

-        while (iter.hasNext()) {
-            c = iter.next();
-            if (!Utils.feq(c.getBottom(), lastTop)) {
-                lastRow = new ArrayList<>();
-                rv.add(lastRow);
+        Float prevY = null;
+        for (Float y : sortedUniqueY) {
+
+            List<Cell> row = new ArrayList<>();
+
+            Float prevX = null;
+            for (Float x : sortedUniqueX) {
+
+                if (prevY != null && prevX != null) {
+                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
+
+                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    if (intersectionCell.isPresent()) {
+                        cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
+                    }
+                    row.add(cell);
+                }
+                prevX = x;
            }
-            lastRow.add(c);
-            lastTop = c.getBottom();
+
+            if (prevY != null && prevX != null) {
+                matrix.add(row);
+            }
+            prevY = y;
        }
-        return rv;
+
+        Collections.reverse(matrix);
+
+        return matrix;
    }


@@ -355,4 +348,4 @@ public class Table extends AbstractTextContainer {
        return sb.toString();
    }

-}
+}
@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@UtilityClass
+public class FileUtils {
+
+    public File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
+
+        File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
+        setRWPermissionsOnlyForOwner(tempFile);
+
+        return tempFile;
+    }
+
+
+    /**
+     * Deletes a file; logs a message with the reason if the deletion fails.
+     * This method is null-safe.
+     * @param file The file to delete.  Can be null.
+     */
+    public void deleteFile(File file) {
+        if (file != null) {
+            try {
+                Files.deleteIfExists(file.toPath());
+            } catch (IOException ex) {
+                log.warn("Could not delete file!", ex);
+            }
+        }
+    }
+
+
+    // We don't need to check the results of the permission setters below,
+    // since we're manipulating a file we created ourselves.
+    @SuppressWarnings({"ResultOfMethodCallIgnored", "squid:S899"})
+    private void setRWPermissionsOnlyForOwner(File tempFile) {
+
+        try {
+            tempFile.setReadable(true, true);
+            tempFile.setWritable(true, true);
+            tempFile.setExecutable(false);
+        } catch (SecurityException ex) {
+            // This should never happen since we're creating a temp file ourselves.
+            log.warn("Caught an exception during temp file creation.  This should not happend.  Check the code.", ex);
+        }
+    }
+
+}
@@ -5,9 +5,12 @@ import java.util.Comparator;
 import java.util.Deque;
 import java.util.List;

+import lombok.experimental.UtilityClass;
+
 /**
 * Copied and minimal modified from PDFBox.
 */
+@UtilityClass
 public final class QuickSort {

    private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
@@ -19,11 +22,6 @@ public final class QuickSort {
    };


-    private QuickSort() {
-
-    }
-
-
    /**
     * Sorts the given list using the given comparator.
     *
@@ -1,14 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.utils;

-import lombok.extern.slf4j.Slf4j;
-
 import java.math.BigDecimal;
 import java.util.Comparator;
 import java.util.List;

+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
+@UtilityClass
@SuppressWarnings("all")
-public class Utils {
+public final class Utils {

    private final static float EPSILON = 0.1f;

@@ -38,3 +40,4 @@ public class Utils {
    }

 }
+
@@ -0,0 +1,503 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.stubbing.Answer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(AbstractTestWithDictionaries.TestConfiguration.class)
+public class AbstractTestWithDictionaries {
+
+    protected static final String RULES = loadFromClassPath("drools/rules.drl");
+    protected static final String RULES_PATH = "drools/rules.drl";
+    protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
+    private static final String VERTEBRATE = "vertebrate";
+    private static final String ADDRESS = "CBI_address";
+    private static final String AUTHOR = "CBI_author";
+    private static final String SPONSOR = "CBI_sponsor";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";
+    private static final String PII = "PII";
+    private static final String ROTATE_SIMPLE = "RotateSimple";
+
+    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    protected final static String TEST_DOSSIER_ID = "123";
+    protected final static String TEST_FILE_ID = "123";
+
+    @Autowired
+    protected StorageService storageService;
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+    @MockBean
+    protected RulesClient rulesClient;
+
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    private final Map<String, List<String>> falsePositive = new HashMap<>();
+    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
+    private final Map<String, String> typeColorMap = new HashMap<>();
+    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
+    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
+    private final Map<String, Integer> rankTypeMap = new HashMap<>();
+    private final Colors colors = new Colors();
+    private final Map<String, Long> reanalysisVersions = new HashMap<>();
+    private final Set<String> deleted = new HashSet<>();
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));
+
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
+    private void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
+                false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                NO_REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
+                false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION,
+                false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
+                false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
+                false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                DOSSIER_REDACTIONS,
+                true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                IMPORTED_REDACTION,
+                true));
+
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        if (entries == null) {
+            entries = Collections.emptyList();
+        }
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
+        });
+        return dictionaryEntries;
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATE, "#ff85f7");
+        typeColorMap.put(ADDRESS, "#ffe187");
+        typeColorMap.put(AUTHOR, "#ffe187");
+        typeColorMap.put(SPONSOR, "#85ebff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
+        typeColorMap.put(HINT_ONLY, "#abc0c4");
+        typeColorMap.put(MUST_REDACT, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
+        typeColorMap.put(TEST_METHOD, "#91fae8");
+        typeColorMap.put(PII, "#66ccff");
+        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");
+        typeColorMap.put(OCR, "#fcc5fb");
+        typeColorMap.put(LOGO, "#ffe187");
+        typeColorMap.put(FORMULA, "#ffe187");
+        typeColorMap.put(SIGNATURE, "#ffe187");
+        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
+
+        hintTypeMap.put(VERTEBRATE, true);
+        hintTypeMap.put(ADDRESS, false);
+        hintTypeMap.put(AUTHOR, false);
+        hintTypeMap.put(SPONSOR, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION, true);
+        hintTypeMap.put(TEST_METHOD, true);
+        hintTypeMap.put(PII, false);
+        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);
+        hintTypeMap.put(OCR, true);
+        hintTypeMap.put(FORMULA, false);
+        hintTypeMap.put(LOGO, false);
+        hintTypeMap.put(SIGNATURE, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS, false);
+        hintTypeMap.put(IMPORTED_REDACTION, false);
+        hintTypeMap.put(ROTATE_SIMPLE, false);
+
+        caseInSensitiveMap.put(VERTEBRATE, true);
+        caseInSensitiveMap.put(ADDRESS, false);
+        caseInSensitiveMap.put(AUTHOR, false);
+        caseInSensitiveMap.put(SPONSOR, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
+        caseInSensitiveMap.put(TEST_METHOD, false);
+        caseInSensitiveMap.put(PII, false);
+        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);
+        caseInSensitiveMap.put(OCR, true);
+        caseInSensitiveMap.put(SIGNATURE, true);
+        caseInSensitiveMap.put(LOGO, true);
+        caseInSensitiveMap.put(FORMULA, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
+        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
+
+        recommendationTypeMap.put(VERTEBRATE, false);
+        recommendationTypeMap.put(ADDRESS, false);
+        recommendationTypeMap.put(AUTHOR, false);
+        recommendationTypeMap.put(SPONSOR, false);
+        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(HINT_ONLY, false);
+        recommendationTypeMap.put(MUST_REDACT, false);
+        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
+        recommendationTypeMap.put(TEST_METHOD, false);
+        recommendationTypeMap.put(PII, false);
+        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);
+        recommendationTypeMap.put(OCR, false);
+        recommendationTypeMap.put(FORMULA, false);
+        recommendationTypeMap.put(SIGNATURE, false);
+        recommendationTypeMap.put(LOGO, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
+        recommendationTypeMap.put(IMPORTED_REDACTION, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE, false);
+
+        rankTypeMap.put(PURITY, 155);
+        rankTypeMap.put(PII, 150);
+        rankTypeMap.put(ADDRESS, 140);
+        rankTypeMap.put(AUTHOR, 130);
+        rankTypeMap.put(SPONSOR, 120);
+        rankTypeMap.put(VERTEBRATE, 110);
+        rankTypeMap.put(MUST_REDACT, 100);
+        rankTypeMap.put(REDACTION_INDICATOR, 90);
+        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
+        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
+        rankTypeMap.put(TEST_METHOD, 60);
+        rankTypeMap.put(HINT_ONLY, 50);
+        rankTypeMap.put(IMAGE, 30);
+        rankTypeMap.put(OCR, 29);
+        rankTypeMap.put(LOGO, 28);
+        rankTypeMap.put(SIGNATURE, 27);
+        rankTypeMap.put(FORMULA, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
+        rankTypeMap.put(IMPORTED_REDACTION, 200);
+        rankTypeMap.put(ROTATE_SIMPLE, 150);
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+
+    protected List<File> getPathsRecursively(File path) {
+
+        List<File> result = new ArrayList<>();
+        if (path == null || path.listFiles() == null) {
+            return result;
+        }
+        for (File f : path.listFiles()) {
+            if (f.isFile()) {
+                result.add(f);
+            } else {
+                result.addAll(getPathsRecursively(f));
+            }
+        }
+        return result;
+
+    }
+
+
+    protected void loadOnlyDictionaryForSimpleFile() {
+
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file) {
+
+        return prepareStorage(file, "files/cv_service_empty_response.json");
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+
+        return request;
+    }
+
+
+    @AfterEach
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class TestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+}
@@ -0,0 +1,213 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.servlet.ServletContext;
+
+import org.apache.commons.collections4.SetUtils;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.core.io.ClassPathResource;
+
+import com.google.common.io.Files;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.Builder;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class AnalysisIntegrationTest extends AbstractTestWithDictionaries {
+
+    @Autowired
+    private AnalyzeService analyzeService;
+
+    @Autowired
+    private ServletContext context;
+
+    @Autowired
+    private RedactionStorageService redactionStorageService;
+
+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+
+    @MockBean
+    private LegalBasisClient legalBasisClient;
+
+
+    @Test
+    public void testAnalysisIntegrationForAllFiles() {
+
+        findAllFilesWithRedactionLogs().forEach((pdfFile, redactionLogFile) -> {
+
+            try {
+                String relativePdfFilePath = new ClassPathResource("").getURI().relativize(pdfFile.toURI()).getPath();
+                log.info("Checking equality for {}", relativePdfFilePath);
+                AnalyzeRequest request = prepareStorage(relativePdfFilePath);
+
+                analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+                analyzeService.analyze(request);
+
+                RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+                Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
+
+                Path entityReferencePath = Paths.get(new ClassPathResource("entities/").getFile().getAbsolutePath().replace("target/test-classes/", "src/test/resources/"),
+                        relativePdfFilePath.replace("files/", "").replace(".pdf", ".json"));
+
+                new File(entityReferencePath.toString().replace(entityReferencePath.getFileName().toString(), "")).mkdirs();
+
+                try (var out = new FileOutputStream(entityReferencePath.toString())) {
+                    ObjectMapperFactory.create().writeValue(out, references);
+                    log.info("{} Entities have been written for File {}", references.size(), relativePdfFilePath);
+                }
+                //RedactionLog originalRedactionLog = ObjectMapperFactory.create().readValue(redactionLogFile, RedactionLog.class);
+                //assertEntitiesAreEqual(originalRedactionLog, redactionLog);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+
+        });
+    }
+
+
+    // only use this if you must replace all entities with newer versions
+    @Test
+    @Disabled
+    @SneakyThrows
+    public void writeNewEntitiesForAllFiles() {
+
+        ClassPathResource fileStem = new ClassPathResource("files/");
+        List<File> allPdfFiles = getAllPdfFilesAsStream(fileStem).toList();
+        System.out.printf("found %d PDF Files to create Entities for\n", allPdfFiles.size());
+        int count = 0;
+        for (File pdfFile : allPdfFiles) {
+            writeNewEntities(pdfFile);
+            System.out.printf("%d|%d finished writing entities for %s\n", count, allPdfFiles.size(), pdfFile.getName());
+            count++;
+        }
+    }
+
+
+    private static Stream<File> getAllPdfFilesAsStream(ClassPathResource fileStem) throws IOException {
+
+        return Arrays.stream(fileStem.getFile().listFiles())
+                .filter(File::isDirectory)
+                .flatMap(file -> Arrays.stream(file.listFiles()))
+                .filter(file -> Files.getFileExtension(file.getName()).equals("pdf"))
+                .filter(File::isFile)
+                .filter(File::canRead);
+    }
+
+
+    @SneakyThrows
+    private void writeNewEntities(File pdfFile) {
+
+        Path relativePdfFilePath = Paths.get(pdfFile.toString().replace(new ClassPathResource("").getFile().getAbsolutePath(), ""));
+        Path relativeFolderPath = relativePdfFilePath.getParent();
+        Path absoluteEntityFolderPath = Paths.get(context.getContextPath(), "src", "test", "resources", "entities", relativeFolderPath.toString().replace("files/", ""));
+        absoluteEntityFolderPath.toFile().mkdirs();
+        Path absoluteEntityFilePath = absoluteEntityFolderPath.resolve(relativePdfFilePath.getFileName().toString().replace("files/", "").replace(".pdf", ".json"));
+
+        AnalyzeRequest request = prepareStorage(relativePdfFilePath.toString());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
+
+        try (var out = new FileOutputStream(absoluteEntityFilePath.toString())) {
+            ObjectMapperFactory.create().writeValue(out, references);
+        }
+
+    }
+
+
+    private void assertEntitiesAreEqual(RedactionLog originalRedactionLog, RedactionLog redactionLog) {
+
+        Set<EntityReference> originalReferences = originalRedactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
+        Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
+
+        Set<EntityReference> missingReferences = SetUtils.difference(originalReferences, references);
+        Set<EntityReference> additionalReferences = SetUtils.difference(references, originalReferences);
+
+        if (missingReferences.size() == 0) {
+            log.info("All original Entities have been found!");
+        } else {
+            log.warn("{} original References not found {}", missingReferences.size(), missingReferences);
+            assert missingReferences.size() == 0;
+        }
+
+        if (additionalReferences.size() == 0) {
+            log.info("No additional Entities have been found!");
+        } else {
+            log.warn("{} additional Entities found", additionalReferences.size());
+            additionalReferences.forEach(e -> log.warn(e.toString()));
+            assert additionalReferences.size() == 0;
+        }
+    }
+
+
+    private EntityReference toReference(RedactionLogEntry redactionLogEntry) {
+
+        return EntityReference.builder()
+                .positions(redactionLogEntry.getPositions())
+                .isRecommendation(redactionLogEntry.isRecommendation())
+                .isFalsePositive(redactionLogEntry.isFalsePositive())
+                .word(redactionLogEntry.getValue())
+                .isHint(redactionLogEntry.isHint())
+                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
+                .reason(redactionLogEntry.getReason())
+                .build();
+    }
+
+
+    @Builder
+    public record EntityReference(
+            String word, String reason, boolean redacted, boolean isHint, boolean isRecommendation, boolean isFalsePositive, boolean isDictionaryEntry, List<Rectangle> positions) {
+
+    }
+
+
+    @SneakyThrows
+    private Map<File, File> findAllFilesWithRedactionLogs() {
+
+        ClassPathResource fileStem = new ClassPathResource("files/");
+        ClassPathResource redactionStem = new ClassPathResource("RedactionLog/files/");
+
+        Map<File, File> fileToRedactionLogFile = new HashMap<>();
+        List<File> allFiles = getAllPdfFilesAsStream(fileStem).toList();
+
+        for (File file : allFiles) {
+            String relativePath = fileStem.getURI().relativize(file.toURI()).getPath().replace(".pdf", ".json");
+            File redactionLogFile = Paths.get(redactionStem.getFile().getAbsolutePath(), relativePath).toFile();
+            if (redactionLogFile.isFile() && redactionLogFile.canRead()) {
+                fileToRedactionLogFile.put(file, redactionLogFile);
+            }
+        }
+        return fileToRedactionLogFile;
+    }
+
+}
@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server;

-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;

-import org.junit.Test;
-import org.junit.runner.RunWith;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -21,16 +21,18 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;

-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DictionaryServiceTest {
@@ -15,6 +15,7 @@ import org.springframework.core.io.InputStreamResource;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;

@@ -67,7 +68,7 @@ public class FileSystemBackedStorageService implements StorageService {
    @SneakyThrows
    public <T> void storeJSONObject(String objectId, T any) {

-        File tempFile = File.createTempFile("test", ".tmp");
+        File tempFile = FileUtils.createTempFile("test", ".tmp");
        getMapper().writeValue(new FileOutputStream(tempFile), any);
        dataMap.put(objectId, tempFile);
    }
@@ -106,7 +107,7 @@ public class FileSystemBackedStorageService implements StorageService {
    @SneakyThrows
    public void storeObject(String objectId, InputStream stream) {

-        File tempFile = File.createTempFile("test", ".tmp");
+        File tempFile = FileUtils.createTempFile("test", ".tmp");

        try (var fileOutputStream = new FileOutputStream(tempFile)) {
            IOUtils.copy(stream, fileOutputStream);
@@ -20,10 +20,10 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.assertj.core.api.Assertions;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -41,7 +41,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -59,8 +59,8 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -72,7 +72,7 @@ import lombok.EqualsAndHashCode;
 import lombok.SneakyThrows;
 import lombok.ToString;

-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(HeadlinesGoldStandardIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class HeadlinesGoldStandardIntegrationTest {
@@ -229,7 +229,7 @@ public class HeadlinesGoldStandardIntegrationTest {
    }


-    @After
+    @AfterEach
    public void cleanupStorage() {

        if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -238,7 +238,7 @@ public class HeadlinesGoldStandardIntegrationTest {
    }


-    @Before
+    @BeforeEach
    public void stubClients() {

        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
@@ -24,11 +24,11 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;

-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -46,7 +46,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -81,8 +81,8 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -92,7 +92,7 @@ import com.iqser.red.storage.commons.service.StorageService;

 import lombok.SneakyThrows;

-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class RedactionIntegrationTest {
@@ -202,7 +202,7 @@ public class RedactionIntegrationTest {
    }


-    @After
+    @AfterEach
    public void cleanupStorage() {

        if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -211,7 +211,7 @@ public class RedactionIntegrationTest {
    }


-    @Before
+    @BeforeEach
    public void stubClients() {

        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
@@ -293,7 +293,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Ignore
+    @Disabled
    public void testLargeScannedFileOOM() {

        AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
@@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
    @Test
    public void titleExtraction() throws IOException {

-        AnalyzeRequest request = prepareStorage("files/RSS/06 - Isopyrazam - Acute Oral Toxicity Rat.pdf");
+        AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");

        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);
@@ -398,7 +398,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Ignore
+    @Disabled
    @SneakyThrows
    public void testIgnoreHint() {

@@ -440,7 +440,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Ignore
+    @Disabled
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

        long start = System.currentTimeMillis();
@@ -1174,7 +1174,7 @@ public class RedactionIntegrationTest {
    public void htmlTablesTest() throws IOException {

        System.out.println("htmlTablesTest");
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/Single Table.pdf");
+        AnalyzeRequest request = prepareStorage("files/new/table-with-merged-cells.pdf");

        RedactionRequest redactionRequest = RedactionRequest.builder()
                .dossierId(request.getDossierId())
@@ -1252,7 +1252,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Ignore
+    @Disabled
    public void resizeRedactionTest() throws IOException {

        String pdfFile = "files/Minimal Examples/Single Table.pdf";
@@ -1369,7 +1369,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Ignore
+    @Disabled
    public void testManualSurroundingText() throws IOException {

        String pdfFile = "files/new/S4.pdf";
@@ -28,11 +28,12 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.apache.commons.lang3.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -49,8 +50,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.annotation.IfProfileValue;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -76,8 +76,8 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -88,7 +88,7 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
@@ -247,7 +247,7 @@ public class RulesTest {
    private LegalBasisClient legalBasisClient;


-    @Before
+    @BeforeEach
    public void stubClients() {

        objectMapper.registerModule(new JavaTimeModule());
@@ -280,7 +280,7 @@ public class RulesTest {
    }


-    @After
+    @AfterEach
    public void cleanupStorage() {

        if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -294,7 +294,7 @@ public class RulesTest {
     * If the RedactionLog already exists, it will be overwritten
     * Test is ignored, because it's for manual tests.
     */
-    @Ignore
+    @Disabled
    @Test
    public void generateRedactionLogForOneFile() {

@@ -336,7 +336,7 @@ public class RulesTest {
     * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
     * If RedactionLog Json for one file does not exist, whole test will fail.
     */
-    @IfProfileValue(name = "test-groups", value = "rules-test")
+    @EnabledIfSystemProperty(named = "test-groups", matches = "(.*)rules-test(.*)")
    @Test
    public void analyseAllFilesAndCompareRedactionLogs() {

@@ -1,23 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

+import java.time.OffsetDateTime;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
+import com.iqser.red.service.redaction.v1.server.utils.MetricValidationUtils;

 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.Set;
-
@Slf4j
 public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest {

@@ -88,10 +89,20 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
 //        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
 //        annotated.getDocument();
 //
-//        File tempFile = File.createTempFile("annotated",".pdf");
+//        File tempFile = FileUtils.createTempFile("annotated",".pdf");
 //        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
 //        log.warn("File saved to: {}",tempFile.getAbsolutePath());
 //        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
+//
+
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getDeepCopyDictionary", 1, null);
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_analyzeDocumentStructure", 1, null);
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getRedactionLog", 1, null);
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_getText", 1, null);
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_reanalyze", 1, null);
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_analyze", 1, null);
+        MetricValidationUtils.validateMetric(prometheusMeterRegistry, "redactmanager_updateDictionary", 1, null);
+
    }

 }
@@ -1,27 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.*;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.when;

-import lombok.SneakyThrows;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -34,18 +28,32 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.when;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
+import lombok.SneakyThrows;

-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class LiveDataIntegrationTest {
@@ -87,6 +95,9 @@ public class LiveDataIntegrationTest {
    @Autowired
    protected RedactionServiceSettings redactionServiceSettings;

+    @Autowired
+    protected PrometheusMeterRegistry prometheusMeterRegistry;
+
    private List<Type> types;

    @Configuration
@@ -104,7 +115,7 @@ public class LiveDataIntegrationTest {


    @SneakyThrows
-    @Before
+    @BeforeEach
    public void prepareTest() {

        when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
@@ -1,11 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.redaction.rulebuilder;

-import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
-
-import org.junit.Test;
-
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
+
 public class RuleBuilderModelServiceTest {

    @Test
@@ -5,7 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import java.util.HashSet;
 import java.util.Set;

-import org.junit.Test;
+import org.junit.jupiter.api.Test;

 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@@ -6,7 +6,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;

 public class RegExPatternTest {

@@ -15,7 +15,7 @@ import org.apache.commons.io.IOUtils;
 import lombok.experimental.UtilityClass;

@UtilityClass
-public class ResourceLoader {
+public final class ResourceLoader {

    public Map<String, String> loadDictionaryFiles() {

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

 import org.assertj.core.api.Assertions;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;

 public class TextNormalizationUtilitiesTest {

@@ -11,8 +11,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;

-import org.junit.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +25,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -49,7 +49,7 @@ import com.iqser.red.storage.commons.service.StorageService;

 import lombok.SneakyThrows;

-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(PdfSegmentationServiceTest.TestConfiguration.class)
 public class PdfSegmentationServiceTest {
@@ -1,24 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.stringmatching;

-import lombok.AllArgsConstructor;
-import lombok.EqualsAndHashCode;
-import lombok.SneakyThrows;
-
-import org.ahocorasick.trie.Trie;
-import org.apache.commons.io.IOUtils;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

 import java.util.HashSet;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;

-@RunWith(SpringRunner.class)
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
 public class StringMatchingPerformanceTest {

    @Test
@@ -34,7 +34,7 @@ public class StringMatchingPerformanceTest {
        System.out.println("Loaded text has a length of " + text.length() + " symbols");
        System.out.println("Dictionary has " + dictionary.size() + " entries");

-        var patterns = dictionary.stream().map(p -> Pattern.compile(Pattern.quote(p))).collect(Collectors.toList());
+        var patterns = dictionary.stream().map(p -> Pattern.compile(Pattern.quote(p))).toList();
        var trie = Trie.builder().ignoreCase().addKeywords(dictionary).build();

        // 1. Naive approach
@@ -56,7 +56,7 @@ public class StringMatchingPerformanceTest {
        t1 = System.currentTimeMillis();
        var boyerMooreIndexes = new HashSet<Index>();
        for (var pattern : patterns) {
-            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).collect(Collectors.toList()));
+            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).toList());
        }
        t2 = System.currentTimeMillis();
        System.out.println("Boyer Moore found " + boyerMooreIndexes.size() + " entries in " + (t2 - t1) + "ms");
@@ -0,0 +1,29 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import java.util.concurrent.TimeUnit;
+
+import io.micrometer.prometheus.PrometheusMeterRegistry;
+import io.micrometer.prometheus.PrometheusTimer;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class MetricValidationUtils {
+
+    public static void validateMetric(PrometheusMeterRegistry registry, String name, int invocationCount, Integer time) {
+
+        var metricOptional = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase(name)).findAny();
+        assertThat(metricOptional.isPresent()).isTrue();
+        PrometheusTimer metric = (PrometheusTimer) metricOptional.get();
+        assertThat(metric.count()).isGreaterThanOrEqualTo(invocationCount);
+        if (time != null) {
+            // for actually timing the method ...
+            assertThat(metric.mean(TimeUnit.MILLISECONDS)).isGreaterThan(time);
+        } else {
+            // everything takes at least one nano-second - test the meter is correctly initialized
+            assertThat(metric.mean(TimeUnit.NANOSECONDS)).isGreaterThan(1);
+        }
+    }
+
+}
@@ -25,4 +25,12 @@ storage:
  bucket-name: 'redaction'
  region: 'us-east-1'
  endpoint: 'https://s3.amazonaws.com'
-  backend: 's3'
+  backend: 's3'
+
+management:
+  endpoint:
+    metrics.enabled: true
+    prometheus.enabled: true
+    health.enabled: true
+  endpoints.web.exposure.include: prometheus, health, metrics
+  metrics.export.prometheus.enabled: true
@@ -13,4 +13,20 @@ Dorn
 Prasher
 David
 annotation
-J.B. RASCLE
+J.B. RASCLE
+（果梗を除去したもの）
+（青森植）
+逸脱：
+（青森植）、ふじ（岩手植）、
+ひろさきふじ（青森植）、ふじ（岩手植）、つがる（長野植須坂）
+ひろさきふじ（青森植）、ふじ（岩手植）、つがる（長野植須坂） 学名
+りんご 品種 ：ひろさきふじ（青森植）、ふじ（岩手植）、つがる（長野植須坂） 学名
+要約
+：準拠
+作物残留試験において、
+日間保存した。
+-20℃
+青森植
+サンプル量
+供試試料 （無処理 区）
+材料
@@ -397,4 +397,11 @@ rule "102: Guidelines FileAttributes"
        Section((text.contains("DATA REQUIREMENT(S):") || text.contains("TEST GUIDELINE(S):")) && (text.contains("OECD") || text.contains("EPA") || text.contains("OPPTS")))
    then
        section.addFileAttribute("OECD Number", "OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", false, 0);
+    end
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        Section(hasTableHeader("h5.1"))
+    then
+        section.redactCell("h5.1", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
    end
Author	SHA1	Message	Date
Kilian Schuettler	1cd6b73d9a	RED-6093: Prototype implementation for document structure and analysis concept *added a new analysis test to ensure the same entities are being found	2023-03-07 18:18:50 +01:00
Viktor Seifert	cab54f6cec	Pull request #517 : RED-6264 Merge in RED/redaction-service from RED-6264 to master * commit 'f9d61a57c1813d34219c6442108af34b411094f7': RED-6264: Corrected metric name and added clarifying comment RED-6264: Implemented metric to show analyze time correlated with pages	2023-03-03 09:47:51 +01:00
Viktor Seifert	f9d61a57c1	RED-6264: Corrected metric name and added clarifying comment	2023-03-02 14:42:14 +01:00
Viktor Seifert	c6837d41af	RED-6264: Implemented metric to show analyze time correlated with pages	2023-03-01 16:39:22 +01:00
Viktor Seifert	8ce093090e	Pull request #516 : RED-6204 Merge in RED/redaction-service from RED-6204 to master * commit '6e73cac99c99fc333b9ec29b4a16aad48ffba149': RED-6204: Added annotation to suppress a false-positive warning in SonarQube RED-6204: Corrected typo in comment RED-6204: Simplified conversion of File to Path RED-6204: Corrected temp file deletion in PdfSegmentationService. RED-6204: Upgraded to newest platform-dependency and migrated tests to Junit5	2023-02-24 17:29:43 +01:00
Viktor Seifert	6e73cac99c	RED-6204: Added annotation to suppress a false-positive warning in SonarQube	2023-02-24 17:19:52 +01:00
Viktor Seifert	dcf040b000	RED-6204: Corrected typo in comment	2023-02-24 15:18:00 +01:00
Viktor Seifert	196cd5934b	RED-6204: Simplified conversion of File to Path	2023-02-24 14:01:57 +01:00
Viktor Seifert	9931b56b92	Merge branch 'master' into RED-6204	2023-02-24 13:55:19 +01:00
Viktor Seifert	6ae4d16bb7	RED-6204: Corrected temp file deletion in PdfSegmentationService. Previously files were not deleted correctly, because the code tried to delete the file while a file-stream was still open.	2023-02-24 13:32:27 +01:00
Viktor Seifert	4b21418163	RED-6204: Upgraded to newest platform-dependency and migrated tests to Junit5	2023-02-24 12:13:43 +01:00
Viktor Seifert	f7ec180710	Pull request #515 : RED-6204 Merge in RED/redaction-service from RED-6204 to master * commit '3fad6381ce71f36083d6f545e1b9c47cecda3ef1': RED-6204: Removed redundant parenthesis RED-6204: Removed redundant variable assignment (sonar issue) & simplified code RED-6204: Remove unused import (sonar issue) RED-6204: Moved code to its own class for metrics. RED-6204: Moved code to its own class for metrics. RED-6204: Remove AspectJ mode setting, since it would require a couple of AspectJ dependencies for a very limited use case RED-6204: Switched to AspectJ to enable proxies on private methods	2023-02-24 10:00:01 +01:00
Viktor Seifert	3fad6381ce	RED-6204: Removed redundant parenthesis	2023-02-23 16:57:58 +01:00
Viktor Seifert	3804acf3c1	RED-6204: Removed redundant variable assignment (sonar issue) & simplified code	2023-02-23 16:49:21 +01:00
Viktor Seifert	8ed976b0f7	RED-6204: Remove unused import (sonar issue)	2023-02-23 16:36:17 +01:00
Viktor Seifert	8fac22b8ab	RED-6204: Moved code to its own class for metrics. Moved a private method to find sections to its own class, so that it can produce a separate metric value.	2023-02-23 11:57:36 +01:00
Viktor Seifert	00ef0eb677	RED-6204: Moved code to its own class for metrics. Moved a private method to find entities to its own class, so that it can produce a separate metric value.	2023-02-22 17:45:30 +01:00
Viktor Seifert	f9b7ad4e3e	RED-6204: Remove AspectJ mode setting, since it would require a couple of AspectJ dependencies for a very limited use case	2023-02-22 16:33:04 +01:00
Viktor Seifert	d65ff273a6	RED-6204: Switched to AspectJ to enable proxies on private methods	2023-02-22 12:39:47 +01:00
Dominique Eiflaender	04343334d8	Pull request #514 : RED-6164: Fixed calculation of image is ocr on scanned pages with cv analysis found tables Merge in RED/redaction-service from RED-6164 to master * commit '0cf867b97c2a240f2bbdca479ebdc36f64ed3bbb': RED-6164: Fixed calculation of image is ocr on scanned pages with cv analysis found tables	2023-02-17 14:36:09 +01:00
deiflaender	0cf867b97c	RED-6164: Fixed calculation of image is ocr on scanned pages with cv analysis found tables	2023-02-17 12:13:39 +01:00
Dominique Eiflaender	f8a0a911bc	Pull request #513 : RED-5664: Enabled to redact words that start or end with seperator, needed for japan documents Merge in RED/redaction-service from RED-5664 to master * commit '1fca62f5782a7b2512092389c13195415bd5df9c': RED-5664: Enabled to redact words that start or end with seperator, needed for japan documents	2023-02-13 10:38:19 +01:00
deiflaender	1fca62f578	RED-5664: Enabled to redact words that start or end with seperator, needed for japan documents	2023-02-13 10:33:22 +01:00
Timo Bejan	0e925f2f24	Pull request #512 : RED-4609 - adjusted some metrics, added tests for metrics Merge in RED/redaction-service from RED-4609 to master * commit 'e23432096cb0ed486a294ed210646bbd8682350e': RED-4609 - adjusted some metrics, added tests for metrics	2023-02-09 09:54:19 +01:00
Timo Bejan	e23432096c	RED-4609 - adjusted some metrics, added tests for metrics	2023-02-08 19:12:47 +02:00
Dominique Eiflaender	c1e2b8da29	Pull request #511 : RED-5276: Fixed strange behavior of text parsing for tables example document Merge in RED/redaction-service from RED-5276-1 to master * commit '16b04b5918a3c8cac0070125d0e984bcd55b9b70': RED-5276: Fixed strange behavior of text parsing for tables example document	2023-01-31 11:16:07 +01:00
deiflaender	16b04b5918	RED-5276: Fixed strange behavior of text parsing for tables example document	2023-01-31 11:03:55 +01:00
Philipp Schramm	c16b6d41d5	Pull request #510 : RED-5248: Fix handling of temp files Merge in RED/redaction-service from RED-5248 to master * commit 'b839c4e3aea67c94fa29d476f63673fc232dcdab': RED-5248: Fix handling of temp files	2023-01-30 12:24:54 +01:00
Philipp Schramm	b839c4e3ae	RED-5248: Fix handling of temp files	2023-01-30 11:51:14 +01:00
Philipp Schramm	6fd6caa8ad	Pull request #509 : RED-5917: Wrong value set for Signature and Logo after Resize Merge in RED/redaction-service from RED-5917 to master * commit '41282c0edd1cbc98357ba04e33731d5b079064b8': RED-5917: Wrong value set for Signature and Logo after Resize	2023-01-19 13:00:30 +01:00
Philipp Schramm	41282c0edd	RED-5917: Wrong value set for Signature and Logo after Resize	2023-01-19 11:25:26 +01:00
Timo Bejan	3c79e65345	Pull request #508 : RED-5981 remove from dictionary pending analhysis Merge in RED/redaction-service from RED-5981 to master * commit 'd2eeaa91a6857b779996e5f5e9254bd4a460ac63': RED-5981 remove from dictionary pending analhysis	2023-01-15 09:15:54 +01:00
Timo Bejan	d2eeaa91a6	RED-5981 remove from dictionary pending analhysis	2023-01-15 16:05:21 +08:00
Dominique Eiflaender	53a375b832	Pull request #507 : RED-5276: Imporved table calculation, support spanned rows and colmns Merge in RED/redaction-service from RED-5276-1 to master * commit 'd233c18d335d17d3c79590dd3a295eaa89881de2': RED-5276: Imporved table calculation, support spanned rows and colmns	2022-12-23 11:25:43 +01:00
deiflaender	d233c18d33	RED-5276: Imporved table calculation, support spanned rows and colmns	2022-12-23 11:20:05 +01:00
Philipp Schramm	b74673ae63	Pull request #506 : RED-5249: Marked utils classes with @UtilityClass Merge in RED/redaction-service from RED-5249 to master * commit 'faa702d3f499b89dfd9a7dedf8c16af89128c77d': RED-5249: Marked utils classes with @UtilityClass	2022-12-15 09:49:06 +01:00
Philipp Schramm	faa702d3f4	RED-5249: Marked utils classes with @UtilityClass	2022-12-14 15:27:23 +01:00
Corina Olariu	05bf95d62b	Pull request #505 : RED-5748 - Redaction Service Error after helm upgrade Merge in RED/redaction-service from RED-5748 to master * commit '776de8392a78f7b26e864e37f4be607d1ff7e48e': RED-5748 - Redaction Service Error after helm upgrade - in case of deleted types throw a NotFoundException providing the info about the type not found instead of NullPointerException	2022-12-14 11:04:10 +01:00
devplant	776de8392a	RED-5748 - Redaction Service Error after helm upgrade - in case of deleted types throw a NotFoundException providing the info about the type not found instead of NullPointerException	2022-12-13 16:39:23 +02:00