Pull request #274: section info port

Merge in RED/redaction-service from section-info to master * commit '980d14b6049c0e7f52c2cc4fffbffa7d5bf54b08': section info port
2021-11-22 08:51:10 +01:00 · 2021-11-22 08:51:10 +01:00 · ab9dcbbafb
commit ab9dcbbafb
parent 1cefa3dc05 980d14b604
5 changed files with 105 additions and 32 deletions
--- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java
+++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java
@ -3,10 +3,9 @@ package com.iqser.red.service.redaction.v1.model;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;

-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;

@Data
@AllArgsConstructor
@ -15,4 +14,16 @@ public class SectionGrid {

    private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();

+    private List<SectionGridSection> sections = new ArrayList<>();
+
+    @Data
+    @RequiredArgsConstructor
+    public static class SectionGridSection {
+
+        private final int sectionNumber;
+        private final String headline;
+        private final Set<Integer> pages;
+        private final List<SectionArea> sectionAreas;
+
+    }
 }
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
@ -25,6 +25,7 @@ import org.springframework.web.bind.annotation.RestController;

 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.stream.Collectors;

@Slf4j
@RestController
@ -157,12 +158,31 @@ public class RedactionController implements RedactionResource {
            throw new NotFoundException("RedactionLog not present");
        }

+        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
+
+        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
+        if (sectionGrid.getSections().isEmpty()) {
+
+            log.info("SectionGrid does not have headlines set. Computing headlines now!");
+            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
+
+            // enhance section grid with headline data
+            for (var sectionText : text.getSectionTexts()) {
+                sectionGrid.getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
+                        sectionText.getHeadline(),
+                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
+                        sectionText.getSectionAreas()));
+            }
+            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
+        }
+
+
        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
        if (redactionLog.getAnalysisVersion() == 0) {
            // old redaction logs are returned directly
            return redactionLog;
        } else {
-            return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
+            return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
        }
    }

--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
@ -10,16 +10,12 @@ import java.util.stream.Stream;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;

-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
@ -78,7 +74,15 @@ public class AnalyzeService {
        List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
        sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);

-        Text text = new Text(pageCount, sectionTexts);
+        Text text = new Text(pageCount,sectionTexts);
+
+        // enhance section grid with headline data
+        sectionTexts.forEach(sectionText -> classifiedDoc
+                .getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
+                        sectionText.getHeadline(),
+                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
+                        sectionText.getSectionAreas())));
+
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
                .getSectionGrid());
--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
@ -1,27 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-
+import com.iqser.red.service.redaction.v1.model.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.time.OffsetDateTime;
+import java.util.*;
+import java.util.stream.Collectors;

@Slf4j
@Service
@ -30,14 +20,16 @@ public class RedactionLogMergeService {

    private final DictionaryService dictionaryService;

+    private final SectionTextService sectionTextService;

-    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId,
+
+    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId,
                                              ManualRedactions manualRedactions, Set<Integer> excludedPages) {

        log.info("Merging Redaction log with manual redactions");
        if (manualRedactions != null) {

-            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
+            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);

            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);

@ -118,6 +110,7 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                    redactionLogEntry.setType(imageRecategorization.getType());
                    redactionLogEntry.setHasBeenRecategorized(true);
+                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
@ -196,6 +189,9 @@ public class RedactionLogMergeService {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
                    redactionLogEntry.setRedacted(true);
+                    if (manualLegalBasisChange.getSection() != null) {
+                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
+                    }
                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
@ -258,7 +254,7 @@ public class RedactionLogMergeService {
    }


-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
+    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
                                                       Map<String, List<Comment>> comments, String dossierTemplateId) {

        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
@ -269,6 +265,9 @@ public class RedactionLogMergeService {
                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
+
+                sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
+
                redactionLogEntries.add(redactionLogEntry);
            }
        }
@ -276,8 +275,8 @@ public class RedactionLogMergeService {
        return redactionLogEntries;
    }

-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions){
-       return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
+    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
+        return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
                .getHeight(), pos.getPage())).collect(Collectors.toList());
    }

--- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java
+++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java
@ -0,0 +1,39 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+public class SectionTextService {
+
+
+    public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.getSection() != null) {
+            // set by UI
+            return;
+        }
+
+        if (sectionGrid != null) {
+            var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
+
+            if (firstPosition != null) {
+
+                for (var section : sectionGrid.getSections()) {
+                    if (section.getPages().contains(firstPosition.getPage())) {
+                        for (var sectionArea : section.getSectionAreas()) {
+                            if (sectionArea.contains(firstPosition)) {
+                                redactionLogEntry.setSection(section.getHeadline());
+                                redactionLogEntry.setSectionNumber(section.getSectionNumber());
+                                return;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}