Pull request #275 : redaction-log info for RED-2659

Merge in RED/redaction-service from section-info to master * commit '892fa6ade0c9f260183686535612e7423ed2ec09': redaction-log info for RED-2659
redaction-log info for RED-2659
2021-11-22 13:49:44 +01:00 · 2021-11-22 14:45:02 +02:00 · 2021-11-22 08:51:10 +01:00 · 2021-11-22 09:46:12 +02:00 · 2021-11-18 10:24:39 +01:00 · 2021-11-18 11:18:34 +02:00
10 changed files with 142 additions and 46 deletions
@@ -13,7 +13,6 @@
    <groupId>com.iqser.red.service</groupId>
    <version>1.0-SNAPSHOT</version>

-
    <packaging>pom</packaging>

    <modules>
@@ -11,6 +11,10 @@

    <artifactId>redaction-service-api-v1</artifactId>

+    <properties>
+        <persistence-service.version>0.103.0</persistence-service.version>
+    </properties>
+
    <dependencies>
        <dependency>
            <groupId>org.springframework</groupId>
@@ -20,7 +24,7 @@
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>persistence-service-api-v1</artifactId>
-            <version>0.77.0</version>
+            <version>${persistence-service.version}</version>
        </dependency>
    </dependencies>
 </project>
@@ -30,6 +30,13 @@ public class RedactionLogEntry {
    private boolean redacted;
    private boolean isHint;
    private boolean isRecommendation;
+
+    private boolean hasBeenResized;
+    private boolean hasBeenRecategorized;
+    private boolean hasLegalBasisChanged;
+    private boolean hasBeenForced;
+    private boolean hasBeenRemovedByManualOverride;
+
    private String section;
    private float[] color;

@@ -3,10 +3,9 @@ package com.iqser.red.service.redaction.v1.model;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;

-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;

@Data
@AllArgsConstructor
@@ -15,4 +14,16 @@ public class SectionGrid {

    private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();

+    private List<SectionGridSection> sections = new ArrayList<>();
+
+    @Data
+    @RequiredArgsConstructor
+    public static class SectionGridSection {
+
+        private final int sectionNumber;
+        private final String headline;
+        private final Set<Integer> pages;
+        private final List<SectionArea> sectionAreas;
+
+    }
 }
@@ -11,6 +11,14 @@

    <artifactId>redaction-service-server-v1</artifactId>

+    <properties>
+        <drools.version>7.59.0.Final</drools.version>
+        <kie.version>7.59.0.Final</kie.version>
+        <locationtech.version>1.16.1</locationtech.version>
+        <pdfbox.jbig2-imageio.version>3.0.3</pdfbox.jbig2-imageio.version>
+        <jai-imageio.version>1.4.0</jai-imageio.version>
+    </properties>
+
    <dependencies>
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
@@ -24,17 +32,17 @@
        <dependency>
            <groupId>org.drools</groupId>
            <artifactId>drools-core</artifactId>
-            <version>7.59.0.Final</version>
+            <version>${drools.version}</version>
        </dependency>
        <dependency>
            <groupId>org.kie</groupId>
            <artifactId>kie-spring</artifactId>
-            <version>7.59.0.Final</version>
+            <version>${kie.version}</version>
        </dependency>
        <dependency>
            <groupId>org.locationtech.jts</groupId>
            <artifactId>jts-core</artifactId>
-            <version>1.16.1</version>
+            <version>${locationtech.version}</version>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
@@ -44,17 +52,17 @@
        <dependency>
            <groupId>org.apache.pdfbox</groupId>
            <artifactId>jbig2-imageio</artifactId>
-            <version>3.0.3</version>
+            <version>${pdfbox.jbig2-imageio.version}</version>
        </dependency>
        <dependency>
            <groupId>com.github.jai-imageio</groupId>
            <artifactId>jai-imageio-core</artifactId>
-            <version>1.4.0</version>
+            <version>${jai-imageio.version}</version>
        </dependency>
        <dependency>
            <groupId>com.github.jai-imageio</groupId>
            <artifactId>jai-imageio-jpeg2000</artifactId>
-            <version>1.4.0</version>
+            <version>${jai-imageio.version}</version>
        </dependency>

        <!-- commons -->
@@ -25,6 +25,7 @@ import org.springframework.web.bind.annotation.RestController;

 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.stream.Collectors;

@Slf4j
@RestController
@@ -157,12 +158,31 @@ public class RedactionController implements RedactionResource {
            throw new NotFoundException("RedactionLog not present");
        }

+        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
+
+        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
+        if (sectionGrid.getSections().isEmpty()) {
+
+            log.info("SectionGrid does not have headlines set. Computing headlines now!");
+            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
+
+            // enhance section grid with headline data
+            for (var sectionText : text.getSectionTexts()) {
+                sectionGrid.getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
+                        sectionText.getHeadline(),
+                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
+                        sectionText.getSectionAreas()));
+            }
+            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
+        }
+
+
        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
        if (redactionLog.getAnalysisVersion() == 0) {
            // old redaction logs are returned directly
            return redactionLog;
        } else {
-            return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
+            return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
        }
    }

@@ -10,16 +10,12 @@ import java.util.stream.Stream;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;

-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
@@ -78,7 +74,15 @@ public class AnalyzeService {
        List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
        sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);

-        Text text = new Text(pageCount, sectionTexts);
+        Text text = new Text(pageCount,sectionTexts);
+
+        // enhance section grid with headline data
+        sectionTexts.forEach(sectionText -> classifiedDoc
+                .getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
+                        sectionText.getHeadline(),
+                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
+                        sectionText.getSectionAreas())));
+
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
                .getSectionGrid());
@@ -1,27 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-
+import com.iqser.red.service.redaction.v1.model.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.time.OffsetDateTime;
+import java.util.*;
+import java.util.stream.Collectors;

@Slf4j
@Service
@@ -30,14 +20,16 @@ public class RedactionLogMergeService {

    private final DictionaryService dictionaryService;

+    private final SectionTextService sectionTextService;

-    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId,
+
+    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId,
                                              ManualRedactions manualRedactions, Set<Integer> excludedPages) {

        log.info("Merging Redaction log with manual redactions");
        if (manualRedactions != null) {

-            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
+            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);

            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);

@@ -53,7 +45,7 @@ public class RedactionLogMergeService {

                if (excludedPages != null && !excludedPages.isEmpty()) {
                    entry.getPositions().forEach(pos -> {
-                        if (excludedPages.contains(pos.getPage())) {
+                        if (!entry.isManual() && excludedPages.contains(pos.getPage())) {
                            entry.setExcluded(true);
                        }
                    });
@@ -117,6 +109,8 @@ public class RedactionLogMergeService {
                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                    redactionLogEntry.setType(imageRecategorization.getType());
+                    redactionLogEntry.setHasBeenRecategorized(true);
+                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
@@ -143,6 +137,7 @@ public class RedactionLogMergeService {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
                            .isRedacted(), true));
+                    redactionLogEntry.setHasBeenRemovedByManualOverride(true);
                } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
@@ -169,6 +164,7 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
                            .isRedacted(), false));
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
+                    redactionLogEntry.setHasBeenForced(true);
                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
@@ -195,6 +191,10 @@ public class RedactionLogMergeService {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
                    redactionLogEntry.setRedacted(true);
+                    redactionLogEntry.setHasLegalBasisChanged(true);
+                    if (manualLegalBasisChange.getSection() != null) {
+                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
+                    }
                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
@@ -220,6 +220,7 @@ public class RedactionLogMergeService {
                            .isRedacted(), false));
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                    redactionLogEntry.setValue(manualResizeRedact.getValue());
+                    redactionLogEntry.setHasBeenResized(true);
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
                } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
@@ -256,7 +257,7 @@ public class RedactionLogMergeService {
    }


-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
+    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
                                                       Map<String, List<Comment>> comments, String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
@@ -267,6 +268,9 @@ public class RedactionLogMergeService {
                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
+
+                sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
+
                redactionLogEntries.add(redactionLogEntry);
            }
        }
@@ -274,8 +278,8 @@ public class RedactionLogMergeService {
        return redactionLogEntries;
    }

-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions){
-       return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
+    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
+        return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
                .getHeight(), pos.getPage())).collect(Collectors.toList());
    }

@@ -298,10 +302,10 @@ public class RedactionLogMergeService {
                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
                .legalBasis(manualRedactionEntry.getLegalBasis())
                .value(manualRedactionEntry.getValue())
+                .section(manualRedactionEntry.getSection())
                .type(manualRedactionEntry.getType())
                .redacted(true)
                .isHint(false)
-                .section(null)
                .sectionNumber(-1)
                .manual(true)
                .status(manualRedactionEntry.getStatus())
@@ -0,0 +1,39 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+public class SectionTextService {
+
+
+    public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.getSection() != null) {
+            // set by UI
+            return;
+        }
+
+        if (sectionGrid != null) {
+            var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
+
+            if (firstPosition != null) {
+
+                for (var section : sectionGrid.getSections()) {
+                    if (section.getPages().contains(firstPosition.getPage())) {
+                        for (var sectionArea : section.getSectionAreas()) {
+                            if (sectionArea.contains(firstPosition)) {
+                                redactionLogEntry.setSection(section.getHeadline());
+                                redactionLogEntry.setSectionNumber(section.getSectionNumber());
+                                return;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
@@ -1195,10 +1195,10 @@ public class RedactionIntegrationTest {

    private static String getTemporaryDirectory() {

-//        String tmpdir = System.getProperty("java.io.tmpdir");
-//        if (StringUtils.isNotBlank(tmpdir)) {
-//            return tmpdir;
-//        }
+        String tmpdir = System.getProperty("java.io.tmpdir");
+        if (StringUtils.isNotBlank(tmpdir)) {
+            return tmpdir;
+        }
        return "/tmp";
    }
Author	SHA1	Message	Date
Timo Bejan	571035dda1	Pull request #275 : redaction-log info for RED-2659 Merge in RED/redaction-service from section-info to master * commit '892fa6ade0c9f260183686535612e7423ed2ec09': redaction-log info for RED-2659	2021-11-22 13:49:44 +01:00
Timo Bejan	892fa6ade0	redaction-log info for RED-2659	2021-11-22 14:45:02 +02:00
Timo Bejan	ab9dcbbafb	Pull request #274 : section info port Merge in RED/redaction-service from section-info to master * commit '980d14b6049c0e7f52c2cc4fffbffa7d5bf54b08': section info port	2021-11-22 08:51:10 +01:00
Timo Bejan	980d14b604	section info port	2021-11-22 09:46:12 +02:00
Timo Bejan	1cefa3dc05	Pull request #273 : 2 new falgs for UI indicators Merge in RED/redaction-service from redaction-log-updates to master * commit 'f80eb6d791f35aa96249ef783e56164db85c8278': 2 new falgs for UI indicators	2021-11-18 10:24:39 +01:00
Timo Bejan	f80eb6d791	2 new falgs for UI indicators	2021-11-18 11:18:34 +02:00
Philipp Schramm	e42d5180ec	Pull request #272 : RED-2430 do not exclude manual entries from excluded pages Merge in RED/redaction-service from feature/RED-2430 to master * commit '3ae8b2cd666eee2fff10e2254cd829084e503893': RED-2430 do not exclude manual entries from excluded pages	2021-11-16 08:53:19 +01:00
Philipp Schramm	3ae8b2cd66	RED-2430 do not exclude manual entries from excluded pages	2021-11-15 16:39:53 +01:00
Philipp Schramm	a6a1d51c17	Pull request #271 : RED-2633 Added section to manual redactions, updated version from persistence-service and extracted versions in pom.xm Merge in RED/redaction-service from RED-2633 to master * commit '6e713c4fcc010e6461e403256c4926974cec2cf8': RED-2633 Added section to manual redactions, updated version from persistence-service and extracted versions in pom.xm	2021-11-15 14:56:52 +01:00
Philipp Schramm	6e713c4fcc	RED-2633 Added section to manual redactions, updated version from persistence-service and extracted versions in pom.xm	2021-11-15 14:36:35 +01:00