Hotfix for duplicate entries in redactionLog

2021-11-30 15:49:29 +01:00
10 changed files with 44 additions and 189 deletions
@@ -18,7 +18,6 @@ public class ManualLegalBasisChange {
    private Status status;
    private String legalBasis;
    private String section;
-    private String value;

    private OffsetDateTime requestDate;
    private OffsetDateTime processedDate;
@@ -24,7 +24,6 @@ public class ManualRedactionEntry {
    private List<Rectangle> positions = new ArrayList<>();
    private Status status;
    private String section;
-    private boolean rectangle;

    private boolean addToDictionary;
    private boolean addToDossierDictionary;
@@ -22,7 +22,6 @@ public class RedactionLogEntry {
    private String type;
    private String value;
    private String reason;
-    private boolean rectangle;
    private int matchedRule;
    private String legalBasis;
    private boolean redacted;
@@ -121,16 +121,6 @@ public class Section {


    public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
-        expandByRegEx(type, pattern, patternCaseInsensitive, group, null);
-    }
-
-
-    public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, String withoutPattern) {
-
-        Pattern compiledWithoutPattern = null;
-        if (withoutPattern != null) {
-            compiledWithoutPattern = Patterns.getCompiledPattern(withoutPattern, patternCaseInsensitive);
-        }

        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);

@@ -141,19 +131,11 @@ public class Section {
                continue;
            }

-            if(withoutPattern != null) {
-                Matcher matcherWithout = compiledWithoutPattern.matcher(entity.getWord());
-                if (matcherWithout.find()) {
-                    continue;
-                }
-            }
-
            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());

            while (matcher.find()) {
                String match = matcher.group(group);
                if (StringUtils.isNotBlank(match)) {
-
                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
                            .getRedactionReason(), entity.getLegalBasis()));

@@ -222,11 +204,6 @@ public class Section {
    }


-    public void ignore(String type){
-        entities.removeIf(entity -> entity.getType().equals(type));
-    }
-
-
    public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group,
                                              String asType) {

@@ -1,15 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
@@ -27,8 +17,16 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;

@Service
@RequiredArgsConstructor
@@ -56,8 +54,7 @@ public class RedactionLogCreatorService {
    }


-    public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, int pageNumber,
-                                                   String dossierTemplateId) {
+    public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, int pageNumber, String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

@@ -84,10 +81,11 @@ public class RedactionLogCreatorService {
                    .sectionNumber(image.getSectionNumber())
                    //.section(image.getSection())
                    // RED-2622
-                    .section("Image:" + image.getType())
+                    .section("Image:"+image.getType())
                    .imageHasTransparency(image.isHasTransparency())
                    .build();

+
            redactionLogEntities.add(redactionLogEntry);
        }

@@ -105,6 +103,7 @@ public class RedactionLogCreatorService {
        entityLoop:
        for (Entity entity : entities.get(page)) {

+
            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {

                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
@@ -124,10 +123,12 @@ public class RedactionLogCreatorService {
                            .flatMap(seq -> seq.getTextPositions().stream())
                            .collect(Collectors.toList()), page);

+
                    redactionLogEntry.getPositions().addAll(rectanglesPerLine);

                }

+
                // FIXME ids should never be null. Figure out why this happens.
                if (redactionLogEntry.getId() != null) {
                    redactionLogEntities.add(redactionLogEntry);
@@ -145,27 +146,16 @@ public class RedactionLogCreatorService {
        if (textPositions.size() == 1) {
            rectangles.add(TextPositionSequence.fromData(textPositions, page).getRectangle());
        } else {
-            float x = textPositions.get(0).getXDirAdj();
            float y = textPositions.get(0).getYDirAdj();
-            float width = textPositions.get(0).getWidth();
-            float height = textPositions.get(0).getHeightDir();
            int startIndex = 0;
-
            for (int i = 1; i < textPositions.size(); i++) {
-                float xDirAdj = textPositions.get(i).getXDirAdj();
                float yDirAdj = textPositions.get(i).getYDirAdj();
-                float widthDir = textPositions.get(i).getWidth();
-                float heightDir = textPositions.get(i).getHeightDir();
-
-                if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) {
+                if (round(yDirAdj,3) != round(y, 3)) {
                    rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, i), page)
                            .getRectangle());
                    y = yDirAdj;
-                    width = widthDir;
-                    height = heightDir;
                    startIndex = i;
                }
-                x = xDirAdj;
            }
            if (startIndex != textPositions.size()) {
                rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page)
@@ -176,32 +166,9 @@ public class RedactionLogCreatorService {
        return rectangles;
    }

-
-    private boolean isCharClose(float x, float xDirAdj, float width) {
-
-        float max = x + (5 * width);
-        if (xDirAdj < max) {
-            return true;
-        }
-        return false;
-    }
-
-
-    private boolean isCharInSameLine(float y, float yCompare, float height, float heightCompare) {
-
-        float offsetHeight = height / 2;
-        float minHeight = height - offsetHeight;
-        float maxHeight = height + offsetHeight;
-
-        float offsetY = height / 10;
-        float minY = y - offsetY;
-        float maxY = y + offsetY;
-
-        if (yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight) {
-            return true;
-        }
-
-        return false;
+    private double round(float value, int decimalPoints) {
+        var d = Math.pow(10, decimalPoints);
+        return Math.round(value * d) / d;
    }


@@ -246,8 +213,8 @@ public class RedactionLogCreatorService {
                    classifiedDoc.getSectionGrid()
                            .getRectanglesPerPage()
                            .computeIfAbsent(page, (x) -> new ArrayList<>())
-                            .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), i + 1, paragraph.getPageBlocks()
-                                    .size()));
+                            .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
+                                    .getHeight(), i + 1, paragraph.getPageBlocks().size()));

                } else if (textBlock instanceof Table) {

@@ -255,7 +222,8 @@ public class RedactionLogCreatorService {
                    for (List<Cell> row : ((Table) textBlock).getRows()) {
                        for (Cell cell : row) {
                            if (cell != null) {
-                                cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell.getWidth(), (float) cell.getHeight()));
+                                cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
+                                        .getWidth(), (float) cell.getHeight()));
                            }
                        }
                    }
@@ -263,8 +231,8 @@ public class RedactionLogCreatorService {
                    classifiedDoc.getSectionGrid()
                            .getRectanglesPerPage()
                            .computeIfAbsent(page, (x) -> new ArrayList<>())
-                            .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), i + 1, paragraph.getPageBlocks()
-                                    .size(), cellRectangles));
+                            .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
+                                    .getHeight(), i + 1, paragraph.getPageBlocks().size(), cellRectangles));

                }
            }
@@ -292,4 +260,5 @@ public class RedactionLogCreatorService {
        return dictionaryService.isRecommendation(type, dossierTemplateId);
    }

+
 }
@@ -190,9 +190,6 @@ public class RedactionLogMergeService {
                    if (manualLegalBasisChange.getSection() != null) {
                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
                    }
-                    if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                        redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-                    }
                } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
                    redactionLogEntry.setStatus(Status.REQUESTED);
@@ -229,8 +226,8 @@ public class RedactionLogMergeService {


    private List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
-                                                        Set<ManualRedactionEntry> manualAdds,
-                                                        Map<String, List<Comment>> comments, String dossierTemplateId) {
+                                                       Set<ManualRedactionEntry> manualAdds,
+                                                       Map<String, List<Comment>> comments, String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();

@@ -273,7 +270,6 @@ public class RedactionLogMergeService {
                .value(manualRedactionEntry.getValue())
                .section(manualRedactionEntry.getSection())
                .type(manualRedactionEntry.getType())
-                .rectangle(manualRedactionEntry.isRectangle())
                .redacted(true)
                .isHint(false)
                .sectionNumber(-1)
@@ -22,7 +22,6 @@ import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;

 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -570,7 +569,7 @@ public class RedactionIntegrationTest {
                .fileId(TEST_FILE_ID)
                .build());

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated3.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated3.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
        }
        long rstart = System.currentTimeMillis();
@@ -683,7 +682,7 @@ public class RedactionIntegrationTest {

        System.out.println("first analysis duration: " + (end - start));

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Test.json")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
            fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
        }

@@ -751,7 +750,7 @@ public class RedactionIntegrationTest {
                .fileId(TEST_FILE_ID)
                .build());

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
        }

@@ -772,7 +771,7 @@ public class RedactionIntegrationTest {
                .fileId(TEST_FILE_ID)
                .build());

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
        }
        long end = System.currentTimeMillis();
@@ -782,32 +781,6 @@ public class RedactionIntegrationTest {
    }


-    @Test
-    public void testIgnoreDossierRedactionIfConfidential() throws IOException {
-
-        long start = System.currentTimeMillis();
-
-        AnalyzeRequest request = prepareStorage("files/new/VV-919901.pdf");
-        AnalyzeResult result = reanalyzeService.analyze(request);
-
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
-            fileOutputStream.write(annotateResponse.getDocument());
-        }
-        long end = System.currentTimeMillis();
-
-        System.out.println("duration: " + (end - start));
-        System.out.println("numberOfPages: " + result.getNumberOfPages());
-    }
-
-
-
-
-
    @Test
    public void testManualRedaction() throws IOException {

@@ -874,7 +847,7 @@ public class RedactionIntegrationTest {
                .fileId(TEST_FILE_ID)
                .build());

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
        }
        long end = System.currentTimeMillis();
@@ -900,7 +873,7 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.classify(redactionRequest);

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Classified.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Classified.pdf")) {
            fileOutputStream.write(result.getDocument());
        }
    }
@@ -922,7 +895,7 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.sections(redactionRequest);

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Sections.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Sections.pdf")) {
            fileOutputStream.write(result.getDocument());
        }
    }
@@ -944,7 +917,7 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.htmlTables(redactionRequest);

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Tables.html")) {
            fileOutputStream.write(result.getDocument());
        }
    }
@@ -966,7 +939,7 @@ public class RedactionIntegrationTest {

        RedactionResult result = redactionController.htmlTables(redactionRequest);

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Tables.html")) {
            fileOutputStream.write(result.getDocument());
        }
    }
@@ -1034,31 +1007,7 @@ public class RedactionIntegrationTest {
                .fileId(TEST_FILE_ID)
                .build());

-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
-            fileOutputStream.write(annotateResponse.getDocument());
-        }
-        long end = System.currentTimeMillis();
-
-        System.out.println("duration: " + (end - start));
-        System.out.println("numberOfPages: " + result.getNumberOfPages());
-    }
-
-
-    @Test
-    public void testExpandByRegEx() throws IOException {
-
-        System.out.println("expandByRegex");
-        long start = System.currentTimeMillis();
-
-        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
-        AnalyzeResult result = reanalyzeService.analyze(request);
-
-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
        }
        long end = System.currentTimeMillis();
@@ -1086,13 +1035,4 @@ public class RedactionIntegrationTest {
        }
    }

-    private static String getTemporaryDirectory() {
-
-        String tmpdir = System.getProperty("java.io.tmpdir");
-        if (StringUtils.isNotBlank(tmpdir)) {
-            return tmpdir;
-        }
-        return "/tmp";
-    }
-
 }
@@ -1,2 +1 @@
-Difenoconazole
-FENPROPIDIN
+Difenoconazole
@@ -7,20 +7,12 @@ global Section section

 // --------------------------------------- CBI rules -------------------------------------------------------------------

-//rule "0: Expand CBI Authors with firstname initials"
-//    when
-//        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
-//    then
-//        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//    end
-
 rule "0: Expand CBI Authors with firstname initials"
    when
        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
    then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
-        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
    end


@@ -7,20 +7,12 @@ global Section section

 // --------------------------------------- CBI rules -------------------------------------------------------------------

-//rule "0: Expand CBI Authors with firstname initials"
-//    when
-//        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
-//    then
-//        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//    end
-
 rule "0: Expand CBI Authors with firstname initials"
    when
        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
    then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
-        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
    end


@@ -344,11 +336,4 @@ rule "29: Redact Dossier Redactions"
         Section(matchesType("dossier_redactions"))
     then
         section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
-     end
-
-rule "30: Ignore dossier_redactions if confidential"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
-    then
-        section.ignore("dossier_redactions");
-    end
+     end