Adding more logs

Adjust rule Do not redact genitive CBI_author
Pull request #124 : Adjusted rules to new requirements, add possibility to addRedaction for rules
2021-02-22 17:10:23 +01:00 · 2021-02-22 11:19:42 +01:00 · 2021-02-18 10:48:26 +01:00 · 2021-02-18 10:33:38 +01:00 · 2021-02-18 09:18:59 +01:00 · 2021-02-18 08:58:51 +01:00
20 changed files with 405 additions and 195 deletions
@@ -72,14 +72,6 @@ public class PlanSpec {
                        .jobs(new Job("Default Job",
                                new BambooKey("JOB1"))
                                .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
                                        new ScriptTask()
                                                .description("Clean")
                                                .inlineBody("#!/bin/bash\n" +
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ManualForceRedact {
+
+    private String id;
+    private String user;
+    private Status status;
+    private String legalBasis;
+
+}
@@ -1,5 +1,5 @@
 package com.iqser.red.service.redaction.v1.model;

 public enum ManualRedactionType {
-    ADD, REMOVE
+    ADD, REMOVE, FORCE_REDACT
 }
@@ -20,6 +20,9 @@ public class ManualRedactions {
    @Builder.Default
    private Set<IdRemoval> idsToRemove = new HashSet<>();

+    @Builder.Default
+    private Set<ManualForceRedact> forceRedacts = new HashSet<>();
+
    @Builder.Default
    private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();

@@ -20,6 +20,7 @@ public class Document {
    private List<Paragraph> paragraphs = new ArrayList<>();
    private List<Header> headers = new ArrayList<>();
    private List<Footer> footers = new ArrayList<>();
+    private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
    private Map<Integer, List<Entity>> entities = new HashMap<>();
    private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
    private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class UnclassifiedText {
+
+    private List<TextBlock> textBlocks;
+
+
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
+        return searchableText;
+    }
+
+}
@@ -111,6 +111,8 @@ public class ClassificationService {
            textBlock.setClassification("TextBlock Italic");
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
            textBlock.setClassification("TextBlock Unknown");
+        } else {
+            textBlock.setClassification("Other");
        }
    }

@@ -51,6 +51,9 @@ public class RedactionController implements RedactionResource {
    @Override
    public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {

+        log.info("Starting redaction analysis...");
+        long startTime = System.currentTimeMillis();
+
        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
            pdDocument.setAllSecurityToBeRemoved(true);

@@ -63,6 +66,8 @@ public class RedactionController implements RedactionResource {
                    .getRuleSetId());

            log.info("Redaction analysis successful...");
+            log.debug("Finishing redaction analysis took: {}.", System.currentTimeMillis() - startTime);
+

            return AnalyzeResult.builder()
                    .sectionGrid(classifiedDoc.getSectionGrid())
@@ -80,12 +85,17 @@ public class RedactionController implements RedactionResource {

    public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {

+        log.info("Starting redaction annotation...");
+        long startTime = System.currentTimeMillis();
+
        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(annotateRequest.getDocument()))) {

            pdDocument.setAllSecurityToBeRemoved(true);
            dictionaryService.updateDictionary(annotateRequest.getRedactionLog().getRuleSetId());
            annotationService.annotate(pdDocument, annotateRequest.getRedactionLog(), annotateRequest.getSectionGrid());

+            log.debug("Finishing redaction annotation took: {}.", System.currentTimeMillis() - startTime);
+
            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
                pdDocument.save(byteArrayOutputStream);
                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
@@ -100,12 +110,17 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {

+        log.info("Starting redaction classification...");
+        long startTime = System.currentTimeMillis();
+
        try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
            pdDocument.setAllSecurityToBeRemoved(true);

            Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
            pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);

+            log.info("Finishing redaction classification took: {}.", System.currentTimeMillis() - startTime);
+
            return convert(pdDocument, classifiedDoc.getPages().size(), pdfSegmentationRequest.getRuleSetId());

        } catch (IOException e) {
@@ -139,6 +139,75 @@ public class Section {
        });
    }

+    public void dismissByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, int ruleNumber, String reason) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    entity.setRedaction(false);
+                    entity.setMatchedRule(ruleNumber);
+                    entity.setRedactionReason(reason);
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
+    public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, String asType) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null));
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
+    public void addHintAnnotationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Matcher matcher = compiledPattern.matcher(searchText);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null);
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+            }
+        }
+    }
+

    public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason, String legalBasis) {

@@ -160,6 +229,13 @@ public class Section {
    }


+    public void addRedaction(String value, String asType, int ruleNumber, String reason, String legalBasis) {
+
+        Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis);
+        EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
+    }
+
+
    public void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
                                String legalBasis) {

@@ -29,8 +29,10 @@ import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;

+import lombok.extern.slf4j.Slf4j;
 import lombok.RequiredArgsConstructor;

+@Slf4j
@Service
@RequiredArgsConstructor
 public class AnnotationService {
@@ -40,6 +42,9 @@ public class AnnotationService {

    public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {

+        log.info("Annotating document...");
+        long startTime = System.currentTimeMillis();
+
        Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);

        for (int page = 1; page <= document.getNumberOfPages(); page++) {
@@ -56,6 +61,8 @@ public class AnnotationService {
                addAnnotations(logEntries, pdPage, page);
            }
        }
+
+        log.info("Finishing document annotation took: {}.", System.currentTimeMillis() - startTime);
    }


@@ -38,6 +38,9 @@ public class DictionaryService {

    public void updateDictionary(String ruleSetId) {

+        log.info("Updating dictionary...");
+        long startTime = System.currentTimeMillis();
+
        long version = dictionaryClient.getVersion(ruleSetId);

        var foundDictionary = dictionariesByRuleSets.get(ruleSetId);
@@ -45,11 +48,16 @@ public class DictionaryService {
        if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
            updateDictionaryEntry(ruleSetId, version);
        }
+
+        log.info("Finishing dictionary update took: {}.", System.currentTimeMillis() - startTime);
    }


    private void updateDictionaryEntry(String ruleSetId, long version) {

+        log.info("Updating dictionary entries...");
+        long startTime = System.currentTimeMillis();
+
        try {
            DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();

@@ -82,10 +90,17 @@ public class DictionaryService {
            log.warn("Got some unknown feignException", e);
            throw e;
        }
+
+        log.info("Finishing dictionary entries update took: {}.", System.currentTimeMillis() - startTime);
+
    }


    public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
+
+        log.info("Updating external dictionary...");
+        long startTime = System.currentTimeMillis();
+
        dictionary.getDictionaryModels().forEach(dm -> {
            if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
                dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
@@ -95,6 +110,9 @@ public class DictionaryService {
                }
            }
        });
+
+        log.info("Finishing external dictionary update took: {}.", System.currentTimeMillis() - startTime);
+
    }


@@ -23,6 +23,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
@@ -50,14 +51,23 @@ public class EntityRedactionService {

    public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {

+        log.info("Processing document...");
+        long startTime = System.currentTimeMillis();
+
        dictionaryService.updateDictionary(ruleSetId);
        KieContainer container = droolsExecutionService.updateRules(ruleSetId);
        long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);

        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);

+        long current = System.currentTimeMillis() - startTime;
+        log.info("Preparing took: {}.", current);
+
        Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));

+        current = System.currentTimeMillis() - startTime - current;
+        log.info("Finding entries took: {}.", current);
+
        if (dictionary.hasLocalEntries()) {

            Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
@@ -71,6 +81,9 @@ public class EntityRedactionService {
            Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
            EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
            EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
+
+            current = System.currentTimeMillis() - startTime - current;
+            log.info("Adding local dictionary annotations took: {}.", current);
        }

        for (Entity entity : documentEntities) {
@@ -89,10 +102,17 @@ public class EntityRedactionService {
            }
        }

+        current = System.currentTimeMillis() - startTime - current;
+        log.info("Adding entity position sequence took: {}.", current);
+
        dictionaryService.updateExternalDictionary(dictionary, ruleSetId);

        classifiedDoc.setDictionaryVersion(dictionary.getVersion());
        classifiedDoc.setRulesVersion(rulesVersion);
+
+        current = System.currentTimeMillis() - startTime - current;
+        log.info("Adjusting rules set took: {}.", current);
+        log.info("Finishing document processing took: {}.", System.currentTimeMillis() - startTime);
    }


@@ -130,6 +150,11 @@ public class EntityRedactionService {
            sectionNumber.incrementAndGet();
        }

+        for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) {
+            sectionSearchableTextPairs.add(processText(unclassifiedText.getSearchableText(), unclassifiedText.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionNumber.incrementAndGet();
+        }
+
        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
            Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
            documentEntities.addAll(analysedRowSection.getEntities());
@@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Comment;
 import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
@@ -34,7 +35,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
@Service
@RequiredArgsConstructor
 public class RedactionLogCreatorService {
@@ -47,6 +50,9 @@ public class RedactionLogCreatorService {
    public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
                                   String ruleSetId) {

+        log.info("Creating redaction log...");
+        long startTime = System.currentTimeMillis();
+
        Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);

        for (int page = 1; page <= numberOfPages; page++) {
@@ -57,14 +63,26 @@ public class RedactionLogCreatorService {
                addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
            }

+            long current = System.currentTimeMillis() - startTime;
+            log.info("Adding entries took: {}.", current);
+
            if (manualRedactionPages.contains(page)) {
                addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
            }

+            current = System.currentTimeMillis() - startTime - current;
+            log.info("Adding manual entries took: {}.", current);
+
            if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
                addImageEntries(classifiedDoc, page, ruleSetId);
            }
+
+            current = System.currentTimeMillis() - startTime - current;
+            log.info("Adding image entries took: {}.", current);
        }
+
+        log.info("Finishing redaction log creation took: {}.", System.currentTimeMillis() - startTime);
+
    }


@@ -136,9 +154,11 @@ public class RedactionLogCreatorService {
                                redactionLogEntry.setRedacted(false);
                                redactionLogEntry.setStatus(Status.APPROVED);
                                manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
                            } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
                                manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
                                redactionLogEntry.setStatus(Status.REQUESTED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
                            } else {
                                redactionLogEntry.setStatus(Status.DECLINED);
                            }
@@ -149,7 +169,34 @@ public class RedactionLogCreatorService {
                            redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
                        }
                    }
+                }

+                if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
+                    for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
+                        if (manualForceRedact.getId().equals(entityPositionSequence.getId())) {
+                            String manualOverrideReason = null;
+                            if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
+                                entity.setRedaction(true);
+                                redactionLogEntry.setRedacted(true);
+                                redactionLogEntry.setStatus(Status.APPROVED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
+                                manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
+                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                            } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
+                                manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
+                                redactionLogEntry.setStatus(Status.REQUESTED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
+                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                            } else {
+                                redactionLogEntry.setStatus(Status.DECLINED);
+                            }
+
+                            entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
+                            redactionLogEntry.setReason(manualOverrideReason);
+                            redactionLogEntry.setManual(true);
+                            redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
+                        }
+                    }
                }

                if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
@@ -39,6 +39,9 @@ public class PdfSegmentationService {

    public Document parseDocument(PDDocument pdDocument) throws IOException {

+        log.info("Parsing document...");
+        long startTime = System.currentTimeMillis();
+
        Document document = new Document();

        List<Page> pages = new ArrayList<>();
@@ -91,6 +94,8 @@ public class PdfSegmentationService {

        sectionsBuilderService.buildSections(document);

+        log.info("Finishing document parsing took: {}.", System.currentTimeMillis() - startTime);
+
        return document;
    }

@@ -15,19 +15,27 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;

+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
@Service
 public class SectionsBuilderService {

    public void buildSections(Document document) {

+        log.debug("Building sections...");
+        long startTime = System.currentTimeMillis();
+
        List<AbstractTextContainer> chunkWords = new ArrayList<>();
        List<Paragraph> chunkBlockList = new ArrayList<>();
        List<Header> headers = new ArrayList<>();
        List<Footer> footers = new ArrayList<>();
+        List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

        AbstractTextContainer prev = null;

@@ -36,6 +44,7 @@ public class SectionsBuilderService {
        for (Page page : document.getPages()) {
            List<TextBlock> header = new ArrayList<>();
            List<TextBlock> footer = new ArrayList<>();
+            List<TextBlock> unclassifiedText = new ArrayList<>();
            for (AbstractTextContainer current : page.getTextBlocks()) {

                if (current.getClassification() == null) {
@@ -54,6 +63,11 @@ public class SectionsBuilderService {
                    continue;
                }

+                if (current.getClassification().equals("Other")) {
+                    unclassifiedText.add((TextBlock) current);
+                    continue;
+                }
+
                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
                        .startsWith("H ") || !document.isHeadlines()) {
                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -78,6 +92,7 @@ public class SectionsBuilderService {
            }
            headers.add(new Header(header));
            footers.add(new Footer(footer));
+            unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
        }

        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -87,6 +102,10 @@ public class SectionsBuilderService {
        document.setParagraphs(chunkBlockList);
        document.setHeaders(headers);
        document.setFooters(footers);
+        document.setUnclassifiedTexts(unclassifiedTexts);
+
+        log.debug("Finishing section building took: {}.", System.currentTimeMillis() - startTime);
+
    }


@@ -23,11 +23,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;

+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
@Service
 public class TableExtractionService {

    public void extractTables(CleanRulings cleanRulings, Page page) {

+        log.debug("Extracting tables...");
+        long startTime = System.currentTimeMillis();
+
        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

        List<TextBlock> toBeRemoved = new ArrayList<>();
@@ -78,6 +84,9 @@ public class TableExtractionService {
        }

        page.getTextBlocks().removeAll(toBeRemoved);
+
+        log.debug("Finishing table extraction took: {}.", System.currentTimeMillis() - startTime);
+
    }


@@ -51,6 +51,7 @@ import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.model.Comment;
 import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Point;
@@ -324,7 +325,6 @@ public class RedactionIntegrationTest {
        recommendationTypeMap.put(PURITY, false);
        recommendationTypeMap.put(IMAGE, false);

-
        rankTypeMap.put(FALSE_POSITIVE, 160);
        rankTypeMap.put(PURITY, 155);
        rankTypeMap.put(PII, 150);
@@ -342,7 +342,6 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
        rankTypeMap.put(IMAGE, 30);

-
        colors.setDefaultColor("#acfc00");
        colors.setNotRedacted("#cccccc");
        colors.setRequestAdd("#04b093");
@@ -418,7 +417,6 @@ public class RedactionIntegrationTest {
    }


-
    private List<File> getPathsRecursively(File path) {

        List<File> result = new ArrayList<>();
@@ -446,7 +444,6 @@ public class RedactionIntegrationTest {
 //        91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
 //        95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf

-
        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
@@ -456,11 +453,13 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -485,11 +484,13 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -504,6 +505,8 @@ public class RedactionIntegrationTest {
    @Test
    public void testManualRedaction() throws IOException {

+//        675eba69b0c2917de55462c817adaa05
+
        System.out.println("testManualRedaction");
        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@@ -518,8 +521,13 @@ public class RedactionIntegrationTest {
                .text("This is a comment test")
                .build();
        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .id("0836727c3508a0b2ea271da69c04cc2f")
-                .status(Status.REQUESTED)
+                .id("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                .status(Status.DECLINED)
+                .build()));
+        manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder()
+                .id("675eba69b0c2917de55462c817adaa05")
+                .legalBasis("Something")
+                .status(Status.APPROVED)
                .build()));

        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
@@ -542,11 +550,13 @@ public class RedactionIntegrationTest {
                .manualRedactions(manualRedactions)
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -640,7 +650,6 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
@@ -662,11 +671,13 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -20,145 +20,69 @@ rule "1: Redact CBI Authors"
    when
        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
    then
-        section.redact("CBI_author", 1, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redact("CBI_author", 1, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "2: Redacted because Section contains Vertebrate"
+rule "2: Redact CBI Address"
    when
-        Section(matchesType("vertebrate"))
+        Section(matchesType("CBI_address") || matchesType("recommendation_CBI_address"))
    then
-        section.redact("CBI_address", 2, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redact("CBI_address", 2, "Address found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "3: Not Redacted because Section contains no Vertebrate"
+rule "3: Do not redact genitive CBI_author"
    when
-        Section(!matchesType("vertebrate"))
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
    then
-        section.redactNot("CBI_address", 3, "No Vertebrate found");
+        section.expandToHintAnnotationByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, "test_method");
+        section.expandToHintAnnotationByRegEx("recommendation_CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, "test_method");
    end


-rule "4: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
-    then
-        section.redactNot("CBI_address", 4, "Vertebrate and No Redaction Indicator found");
-    end
-
-
-rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
-    then
-        section.redact("CBI_address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "6: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
+rule "4: Redact Author(s) cells in Tables with Author(s) header"
    when
        Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
    then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "7: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
+rule "5: Redact Author cells in Tables with Author header"
    when
        Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
    then
-        section.redactCell("Author", 7, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "8: Not redacted because Vertebrate Study = N"
+rule "6: Redact and recommand Authors in Tables with Vertebrate study Y/N header"
    when
-        Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
+        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
    then
-        section.redactCell("Author(s)", 8, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redactNot("CBI_address", 8, "Not redacted because row is not a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
+        section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "9: Redact if must redact entry is found"
-    when
-        Section(matchesType("must_redact"))
-    then
-        section.redact("CBI_address", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
-    then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 10, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
-    end
-
-
-rule "11: Redact sponsor company"
-    when
-        Section(searchText.toLowerCase().contains("batches produced at"))
-    then
-        section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 11, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("batches produced at", "must_redact");
-    end
-
-
-rule "12: Redact determination of residues"
-    when
-        Section((
-            searchText.toLowerCase.contains("determination of residues") ||
-            searchText.toLowerCase.contains("determination of total residues")
-        ) && (
-            searchText.toLowerCase.contains("livestock") ||
-            searchText.toLowerCase.contains("live stock") ||
-            searchText.toLowerCase.contains("tissue") ||
-            searchText.toLowerCase.contains("tissues") ||
-            searchText.toLowerCase.contains("liver") ||
-            searchText.toLowerCase.contains("muscle") ||
-            searchText.toLowerCase.contains("bovine") ||
-            searchText.toLowerCase.contains("ruminant") ||
-            searchText.toLowerCase.contains("ruminants")
-        ))
-    then
-        section.redact("CBI_address", 12, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("determination of residues", "must_redact");
-        section.addHintAnnotation("determination of total residues", "must_redact");
-        section.addHintAnnotation("livestock", "must_redact");
-        section.addHintAnnotation("live stock", "must_redact");
-        section.addHintAnnotation("tissue", "must_redact");
-        section.addHintAnnotation("tissues", "must_redact");
-        section.addHintAnnotation("liver", "must_redact");
-        section.addHintAnnotation("muscle", "must_redact");
-        section.addHintAnnotation("bovine", "must_redact");
-        section.addHintAnnotation("ruminant", "must_redact");
-        section.addHintAnnotation("ruminants", "must_redact");
-    end
-
-
-rule "13: Redact if CTL/* or BL/*  was found"
+rule "7: Redact if CTL/* or BL/*  was found"
    when
        Section(searchText.contains("CTL/") || searchText.contains("BL/"))
    then
-        section.redact("CBI_address", 13, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("CTL", "must_redact");
-        section.addHintAnnotation("BL", "must_redact");
+        section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
+        section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
    end


-rule "14: Redact and add recommendation for et al. author"
+rule "8: Redact and add recommendation for et al. author"
    when
        Section(searchText.contains("et al"))
    then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
    end


-rule "15: Add recommendation for Addresses in Test Organism sections"
+rule "9: Add recommendation for Addresses in Test Organism sections"
    when
        Section(searchText.contains("Species:") && searchText.contains("Source:"))
    then
@@ -166,7 +90,7 @@ rule "15: Add recommendation for Addresses in Test Organism sections"
    end


-rule "16: Add recommendation for Addresses in Test Animals sections"
+rule "10: Add recommendation for Addresses in Test Animals sections"
    when
        Section(searchText.contains("Species") && searchText.contains("Source"))
    then
@@ -174,34 +98,35 @@ rule "16: Add recommendation for Addresses in Test Animals sections"
    end


-rule "17: Do not redact Names and Addresses if no redaction Indicator is contained"
+rule "11: Do not redact Names and Addresses if Published Information found"
    when
        Section(matchesType("published_information"))
    then
-        section.redactNot("CBI_author", 17, "Vertebrate and Published Information found");
-        section.redactNot("CBI_address", 17, "Vertebrate and Published Information found");
+        section.redactNot("CBI_author", 11, "Published Information found");
+        section.redactNot("CBI_address", 11, "Published Information found");
    end

+
 // --------------------------------------- PII rules -------------------------------------------------------------------


-rule "18: Redacted PII Personal Identification Information"
+rule "12: Redacted PII Personal Identification Information"
    when
        Section(matchesType("PII"))
    then
-        section.redact("PII", 18, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redact("PII", 12, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "19: Redact Emails by RegEx"
+rule "13: Redact Emails by RegEx"
    when
        Section(searchText.contains("@"))
    then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 19, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "20: Redact contact information"
+rule "14: Redact contact information"
    when
       Section(text.contains("Contact point:")
       || text.contains("Phone:")
@@ -217,108 +142,120 @@ rule "20: Redact contact information"
       || text.contains("Telephone No:")
       || text.contains("Fax number:")
       || text.contains("Telephone:")
+       || text.contains("Phone No.")
       || text.contains("European contact:"))
    then
-          section.redactLineAfter("Contact point:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Phone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Email:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("e-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail address:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Alternative contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone No:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("No:", "Fax", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("Contact:", "Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("European contact:",  "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+          section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end


-rule "21: Redact contact information if applicant is found"
+rule "15: Redact contact information if applicant is found"
    when
        Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
    then
-        section.redactLineAfter("Contact point:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Email:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("e-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail address:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Alternative contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone No:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("Contact:", "Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("European contact:",  "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "22: Redact contact information if Producer is found"
+rule "16: Redact contact information if Producer is found"
    when
        Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
    then
-        section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "23: Redact AUTHOR(S)"
+rule "17: Redact AUTHOR(S)"
    when
        Section(searchText.contains("AUTHOR(S):"))
    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 23, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "24: Redact PERFORMING LABORATORY"
+rule "18: Redact PERFORMING LABORATORY"
    when
        Section(searchText.contains("PERFORMING LABORATORY:"))
    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 24, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "25: Redact On behalf of Sequani Ltd.:"
+rule "19: Redact On behalf of Sequani Ltd.:"
    when
        Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
    then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 25, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "26: Redact On behalf of Syngenta Ltd.:"
+rule "20: Redact On behalf of Syngenta Ltd.:"
    when
        Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
    then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 26, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


 // --------------------------------------- other rules -------------------------------------------------------------------

-rule "27: Redact Purity"
+rule "21: Purity Hint"
    when
-        Section(searchText.contains("purity"))
+        Section(searchText.toLowerCase().contains("purity"))
    then
-	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 27, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "22: Redact Must Redact"
+    when
+        Section(matchesType("must_redact"))
+    then
+        section.redact("must_redact", 22, "Must Redact found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end
@@ -10,7 +10,7 @@
        <Root level="warn">
            <AppenderRef ref="CONSOLE" />
        </Root>
-        <Logger name="com.iqser.gin4" level="info" />
+        <Logger name="com.iqser" level="info" />
    </Loggers>

 </Configuration>
Author	SHA1	Message	Date
lmaldacker	2ba2340081	Adding more logs	2021-02-22 17:10:23 +01:00
lmaldacker	27c90be145	Adjust rule Do not redact genitive CBI_author	2021-02-22 11:19:42 +01:00
Dominique Eiflaender	7d507c8fda	Pull request #124 : Adjusted rules to new requirements, add possibility to addRedaction for rules Merge in RED/redaction-service from NewRequirementsRules to master * commit 'b6a13d1ff8a55f1b5ef528af5ea726b5e17b4950': Adjusted rules to new requirements, add possibility to addRedaction for rules	2021-02-18 10:48:26 +01:00
Dominique Eifländer	b6a13d1ff8	Adjusted rules to new requirements, add possibility to addRedaction for rules	2021-02-18 10:33:38 +01:00
Dominique Eiflaender	06e418c34e	Pull request #123 : RED-1070: Fixed not finding annotation on not classified textblocks Merge in RED/redaction-service from RED-1070 to master * commit 'de1dea7ac323dc005baeb0660cc492ab84a88164': RED-1070: Fixed not finding annotation on not classified textblocks	2021-02-18 09:18:59 +01:00
Dominique Eifländer	de1dea7ac3	RED-1070: Fixed not finding annotation on not classified textblocks	2021-02-18 08:58:51 +01:00
Dominique Eiflaender	53509f072e	Pull request #122 : RED-1055: Enabled to force redact ignore annotations Merge in RED/redaction-service from RED-1055 to master * commit 'e0fba8d38ce9858235ca84182f4587d472890ef1': RED-1055: Enabled to force redact ignore annotations	2021-02-12 10:55:04 +01:00
Dominique Eifländer	e0fba8d38c	RED-1055: Enabled to force redact ignore annotations	2021-02-12 10:41:32 +01:00
Lena Maldacker	8989191314	Pull request #121 : Add hint annotation by reg-ex and expand to hint annotation by reg-ex Merge in RED/redaction-service from dismissGen to master * commit '0f263c69b8f2eb7201f062cc263c8d06b7a710a1': Add hint annotation by reg-ex and expand to hint annotation by reg-ex	2021-02-11 17:22:55 +01:00
lmaldacker	0f263c69b8	Add hint annotation by reg-ex and expand to hint annotation by reg-ex	2021-02-11 16:53:07 +01:00
Kresnadi Budisantoso	ea1c2898bc	Remove duplicate build tasks in bamboo specs	2021-02-11 14:11:22 +01:00