From 980d14b6049c0e7f52c2cc4fffbffa7d5bf54b08 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Mon, 22 Nov 2021 09:46:12 +0200 Subject: [PATCH] section info port --- .../redaction/v1/model/SectionGrid.java | 17 ++++++-- .../controller/RedactionController.java | 22 +++++++++- .../redaction/service/AnalyzeService.java | 18 ++++---- .../service/RedactionLogMergeService.java | 41 +++++++++---------- .../redaction/service/SectionTextService.java | 39 ++++++++++++++++++ 5 files changed, 105 insertions(+), 32 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java index 362b5c5c..d1f027a1 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java @@ -3,10 +3,9 @@ package com.iqser.red.service.redaction.v1.model; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Data @AllArgsConstructor @@ -15,4 +14,16 @@ public class SectionGrid { private Map> rectanglesPerPage = new HashMap<>(); + private List sections = new ArrayList<>(); + + @Data + @RequiredArgsConstructor + public static class SectionGridSection { + + private final int sectionNumber; + private final String headline; + private final Set pages; + private final List sectionAreas; + + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index df773686..0c882390 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -25,6 +25,7 @@ import org.springframework.web.bind.annotation.RestController; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.stream.Collectors; @Slf4j @RestController @@ -157,12 +158,31 @@ public class RedactionController implements RedactionResource { throw new NotFoundException("RedactionLog not present"); } + log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion()); + + SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId()); + if (sectionGrid.getSections().isEmpty()) { + + log.info("SectionGrid does not have headlines set. Computing headlines now!"); + var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId()); + + // enhance section grid with headline data + for (var sectionText : text.getSectionTexts()) { + sectionGrid.getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), + sectionText.getHeadline(), + sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), + sectionText.getSectionAreas())); + } + redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); + } + + log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion()); if (redactionLog.getAnalysisVersion() == 0) { // old redaction logs are returned directly return redactionLog; } else { - return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages()); + return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index ce91150e..7fa2721e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -10,16 +10,12 @@ import java.util.stream.Stream; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.Rectangle; import org.kie.api.runtime.KieContainer; import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.RequestBody; -import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; -import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; @@ -78,7 +74,15 @@ public class AnalyzeService { List sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc); sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); - Text text = new Text(pageCount, sectionTexts); + Text text = new Text(pageCount,sectionTexts); + + // enhance section grid with headline data + sectionTexts.forEach(sectionText -> classifiedDoc + .getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), + sectionText.getHeadline(), + sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), + sectionText.getSectionAreas()))); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc .getSectionGrid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 702fa22c..9a46d3ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,27 +1,17 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.model.ManualRedactionType; -import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; - +import com.iqser.red.service.redaction.v1.model.*; import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Service @@ -30,14 +20,16 @@ public class RedactionLogMergeService { private final DictionaryService dictionaryService; + private final SectionTextService sectionTextService; - public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, + + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId, ManualRedactions manualRedactions, Set excludedPages) { log.info("Merging Redaction log with manual redactions"); if (manualRedactions != null) { - var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); + var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); @@ -118,6 +110,7 @@ public class RedactionLogMergeService { redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setType(imageRecategorization.getType()); redactionLogEntry.setHasBeenRecategorized(true); + redactionLogEntry.setSection("Image:" + redactionLogEntry.getType()); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); @@ -196,6 +189,9 @@ public class RedactionLogMergeService { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); redactionLogEntry.setRedacted(true); + if (manualLegalBasisChange.getSection() != null) { + redactionLogEntry.setSection(manualLegalBasisChange.getSection()); + } } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); @@ -258,7 +254,7 @@ public class RedactionLogMergeService { } - public List addManualAddEntries(Set manualAdds, + public List addManualAddEntries(SectionGrid sectionGrid, Set manualAdds, Map> comments, String dossierTemplateId) { List redactionLogEntries = new ArrayList<>(); @@ -269,6 +265,9 @@ public class RedactionLogMergeService { RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId); redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions())); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId())); + + sectionTextService.handleSectionText(sectionGrid, redactionLogEntry); + redactionLogEntries.add(redactionLogEntry); } } @@ -276,8 +275,8 @@ public class RedactionLogMergeService { return redactionLogEntries; } - private List convertPositions(List positions){ - return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos + private List convertPositions(List positions) { + return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos .getHeight(), pos.getPage())).collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java new file mode 100644 index 00000000..ac718724 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java @@ -0,0 +1,39 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionGrid; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class SectionTextService { + + + public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) { + + if (redactionLogEntry.getSection() != null) { + // set by UI + return; + } + + if (sectionGrid != null) { + var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null; + + if (firstPosition != null) { + + for (var section : sectionGrid.getSections()) { + if (section.getPages().contains(firstPosition.getPage())) { + for (var sectionArea : section.getSectionAreas()) { + if (sectionArea.contains(firstPosition)) { + redactionLogEntry.setSection(section.getHeadline()); + redactionLogEntry.setSectionNumber(section.getSectionNumber()); + return; + } + } + } + } + } + } + } +}