From b8dc0e448d9101db8099aaf638ac929a16c87c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 15 Jun 2021 12:29:46 +0200 Subject: [PATCH 01/13] RED-1472: Fixed image merging in rotated pages --- .../segmentation/ImageMergeService.java | 165 ++++++++++++++++ .../segmentation/PdfSegmentationService.java | 179 +++--------------- 2 files changed, 187 insertions(+), 157 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java new file mode 100644 index 00000000..73a94909 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java @@ -0,0 +1,165 @@ +package com.iqser.red.service.redaction.v1.server.segmentation; + +import java.awt.Graphics; +import java.awt.geom.Rectangle2D; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageMergeService { + + + public List mergeImages(List images, int rotation){ + + List mergedList = processImages(images, rotation); + + List imagesInImage = new ArrayList<>(); + for(PdfImage image: mergedList){ + for (PdfImage inner: mergedList){ + if(image != inner && image.getPosition().contains(inner.getPosition().getX(), inner.getPosition().getY(), inner.getPosition().getWidth(), inner.getPosition().getHeight())){ + imagesInImage.add(inner); + } + } + } + mergedList.removeAll(imagesInImage); + + return mergedList; + } + + + //merge images, if they are separated during pdf import, return new list of Pdfimages + private List processImages(List imageList, int rotation) { + if (imageList.size() > 1) { + List mergedList = new ArrayList<>(); + int countElementsInList = 0; + boolean beginImage = true; + + // a List of Boolean, true = candidate for merging, false = no merging + List candidatesList = getCandidatesList(imageList, rotation); + + // loop through list, if there are candidates for merging (true), merge images and add it to mergedList + for (int i = 0; i < candidatesList.size(); i++) { + if (candidatesList.get(i)) { + if (beginImage) { + //begin of image, merge two parts of imageList + PdfImage mergedImage = mergeTwoImages(imageList.get(i), imageList.get(i + 1), rotation); + // image merge successful + if (mergedImage != null) { + mergedList.add(mergedImage); + countElementsInList++; + } + } else { + //middle of an image, merge current piece auf mergedList with image of imageList + PdfImage mergedImage = mergeTwoImages(mergedList.get(countElementsInList - 1), imageList.get(i + 1), rotation); + // image merge successful + if (mergedImage != null) { + mergedList.set(countElementsInList - 1, mergedImage); + } + } + beginImage = false; + } else { + // if the last candidate is false, then both images i and i+1 must be added + if (i == candidatesList.size() - 1) { + if (countElementsInList > 0 && mergedList.get(countElementsInList - 1) == imageList.get(i)) { + mergedList.add(imageList.get(i + 1)); + } else { + mergedList.add(imageList.get(i)); + mergedList.add(imageList.get(i + 1)); + } + } else { + //first image is not splitted, add i to resultlist + if (beginImage) { + mergedList.add(imageList.get(i)); + countElementsInList++; + } else { + // i is the end of an image, add begin of new image + mergedList.add(imageList.get(i + 1)); + countElementsInList++; + beginImage = false; + } + } + } + } + return mergedList; + } else { + return imageList; + } + } + + private PdfImage mergeTwoImages(PdfImage image1, PdfImage image2, int rotation) { + + // diese Angaben von getPosition scheinen nicht richtig zu sein, damit werden teile des Bildes abgeschnitten + double width = image1.getPosition().getWidth(); + double width2 = image2.getPosition().getWidth(); + double height1 = image1.getPosition().getHeight(); + double height2 = image2.getPosition().getHeight(); + // mit den Werten, die unter Image gespeichert sind, funktioniert es + double img1height = image1.getImage().getHeight(); + double img1width = image1.getImage().getWidth(); + double img2height = image2.getImage().getHeight(); + + BufferedImage mergedImage = new BufferedImage((int) img1width, (int) (img1height + img2height), BufferedImage.TYPE_INT_RGB); + Graphics mergedImageGraphics = mergedImage.getGraphics(); + try { + mergedImageGraphics.drawImage(image1.getImage(), 0, 0, null); + mergedImageGraphics.drawImage(image2.getImage(), 0, (int) (img1height), null); + + // set Image, Position and type for merged Image + //set position for merged image with values of image1 and the height of both + Rectangle2D pos = new Rectangle2D.Float(); + pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), rotation == 90 ? width + width2: width, rotation == 90 ? height1 : height1 + height2); + PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage()); + // Graphics need to be disposed + + image1.getImage().flush(); + image2.getImage().flush(); + + mergedImage.flush(); + mergedImageGraphics.dispose(); + + return newPdfImage; + } catch (Exception e) { + // failed to merge image + log.error("Failed to merge image", e); + return null; + } + + + } + + //make a list of true and false, if the image is a candidate for merging + private List getCandidatesList(List imageList, int rotation) { + List candidatesList = new ArrayList<>(); + for (int i = 0; i < imageList.size(); i++) { + if (i >= 1) { + candidatesList.add(isCandidateForMerging(imageList.get(i - 1), imageList.get(i), rotation)); + } + } + return candidatesList; + } + + // evaluate if two images are candidates for merging, depending on their coordinates, width and height + private boolean isCandidateForMerging(PdfImage image1, PdfImage image2, int rotation) { + double x1 = rotation == 90 ? image1.getPosition().getY() : image1.getPosition().getX(); + double y1 = rotation == 90 ? image1.getPosition().getX() : image1.getPosition().getY(); + double width1 = rotation == 90 ? image1.getPosition().getHeight() : image1.getPosition().getWidth(); + double x2 = rotation == 90 ? image2.getPosition().getY() : image2.getPosition().getX(); + double y2 = rotation == 90 ? image2.getPosition().getX() : image2.getPosition().getY(); + double width2 = rotation == 90 ? image2.getPosition().getHeight() : image2.getPosition().getWidth(); + double height2 = rotation == 90 ? image2.getPosition().getWidth() : image2.getPosition().getHeight(); + //if the x-coordinates and widths of images are equal and the height is equal to difference between y-coordinates, + // then it is the same picture and has to be merged -> return true + return x1 == x2 && width1 == width2 && Math.ceil(height2) == Math.ceil(rotation == 90 ? y2 - y1 : y1 - y2) && width2 > (height2 / 6); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index a33a009d..22643b0b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -1,6 +1,19 @@ package com.iqser.red.service.redaction.v1.server.segmentation; -import com.iqser.red.service.redaction.v1.model.Rectangle; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.pdfbox.io.MemoryUsageSetting; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.springframework.stereotype.Service; + import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; @@ -15,24 +28,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService; import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.apache.pdfbox.io.MemoryUsageSetting; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.common.PDRectangle; -import org.springframework.stereotype.Service; - -import java.awt.Graphics; -import java.awt.geom.Rectangle2D; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; @Slf4j @Service @@ -47,13 +45,17 @@ public class PdfSegmentationService { private final ClassificationService classificationService; private final SectionsBuilderService sectionsBuilderService; private final ImageClassificationService imageClassificationService; + private final ImageMergeService imageMergeService; public Document parseDocument(InputStream documentInputStream) throws IOException { + return parseDocument(documentInputStream, false); } + public Document parseDocument(InputStream documentInputStream, boolean ignoreImages) throws IOException { + PDDocument pdDocument = null; try { //create tempFile @@ -64,7 +66,6 @@ public class PdfSegmentationService { Document document = new Document(); List pages = new ArrayList<>(); - pdDocument = reinitializePDDocument(tempFile, null); long pageCount = pdDocument.getNumberOfPages(); @@ -101,32 +102,19 @@ public class PdfSegmentationService { page.setRotation(rotation); page.setLandscape(isLandscape || isRotated); page.setPageNumber(pageNumber); - List mergedList = processImages(stripper.getImages()); - - List imagesInImage = new ArrayList<>(); - for(PdfImage image: mergedList){ - for (PdfImage inner: mergedList){ - if(image != inner && image.getPosition().contains(inner.getPosition().getX(), inner.getPosition().getY(), inner.getPosition().getWidth(), inner.getPosition().getHeight())){ - imagesInImage.add(inner); - } - } - } - mergedList.removeAll(imagesInImage); + List mergedList = imageMergeService.mergeImages(stripper.getImages(), rotation); page.setImages(mergedList); tableExtractionService.extractTables(cleanRulings, page); buildPageStatistics(page); increaseDocumentStatistics(page, document); - if (!ignoreImages) { imageClassificationService.classifyImages(page); } pages.add(page); - - } document.setPages(pages); @@ -149,7 +137,9 @@ public class PdfSegmentationService { } } + private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException { + if (pdDocument != null) { pdDocument.close(); } @@ -164,130 +154,6 @@ public class PdfSegmentationService { return newPDDocument; } - //merge images, if they are separated during pdf import, return new list of Pdfimages - private List processImages(List imageList) { - if (imageList.size() > 1) { - List mergedList = new ArrayList<>(); - int countElementsInList = 0; - boolean beginImage = true; - - // a List of Boolean, true = candidate for merging, false = no merging - List candidatesList = getCandidatesList(imageList); - - // loop through list, if there are candidates for merging (true), merge images and add it to mergedList - for (int i = 0; i < candidatesList.size(); i++) { - if (candidatesList.get(i)) { - if (beginImage) { - //begin of image, merge two parts of imageList - PdfImage mergedImage = mergeTwoImages(imageList.get(i), imageList.get(i + 1)); - // image merge successful - if (mergedImage != null) { - mergedList.add(mergedImage); - countElementsInList++; - } - } else { - //middle of an image, merge current piece auf mergedList with image of imageList - PdfImage mergedImage = mergeTwoImages(mergedList.get(countElementsInList - 1), imageList.get(i + 1)); - // image merge successful - if (mergedImage != null) { - mergedList.set(countElementsInList - 1, mergedImage); - } - } - beginImage = false; - } else { - // if the last candidate is false, then both images i and i+1 must be added - if (i == candidatesList.size() - 1) { - if (countElementsInList > 0 && mergedList.get(countElementsInList - 1) == imageList.get(i)) { - mergedList.add(imageList.get(i + 1)); - } else { - mergedList.add(imageList.get(i)); - mergedList.add(imageList.get(i + 1)); - } - } else { - //first image is not splitted, add i to resultlist - if (beginImage) { - mergedList.add(imageList.get(i)); - countElementsInList++; - } else { - // i is the end of an image, add begin of new image - mergedList.add(imageList.get(i + 1)); - countElementsInList++; - beginImage = false; - } - } - } - } - return mergedList; - } else { - return imageList; - } - } - - private PdfImage mergeTwoImages(PdfImage image1, PdfImage image2) { - - // diese Angaben von getPosition scheinen nicht richtig zu sein, damit werden teile des Bildes abgeschnitten - double width = image1.getPosition().getWidth(); - double height1 = image1.getPosition().getHeight(); - double height2 = image2.getPosition().getHeight(); - // mit den Werten, die unter Image gespeichert sind, funktioniert es - double img1height = image1.getImage().getHeight(); - double img1width = image1.getImage().getWidth(); - double img2height = image2.getImage().getHeight(); - - BufferedImage mergedImage = new BufferedImage((int) img1width, (int) (img1height + img2height), BufferedImage.TYPE_INT_RGB); - Graphics mergedImageGraphics = mergedImage.getGraphics(); - try { - mergedImageGraphics.drawImage(image1.getImage(), 0, 0, null); - mergedImageGraphics.drawImage(image2.getImage(), 0, (int) (img1height), null); - - // set Image, Position and type for merged Image - //set position for merged image with values of image1 and the height of both - Rectangle2D pos = new Rectangle2D.Float(); - pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), width, height1 + height2); - PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage()); - // Graphics need to be disposed - - image1.getImage().flush(); - image2.getImage().flush(); - - mergedImage.flush(); - mergedImageGraphics.dispose(); - - return newPdfImage; - } catch (Exception e) { - // failed to merge image - log.error("Failed to merge image", e); - return null; - } - - - } - - //make a list of true and false, if the image is a candidate for merging - private List getCandidatesList(List imageList) { - List candidatesList = new ArrayList<>(); - for (int i = 0; i < imageList.size(); i++) { - if (i >= 1) { - candidatesList.add(isCandidateForMerging(imageList.get(i - 1), imageList.get(i))); - } - } - return candidatesList; - } - - // evaluate if two images are candidates for merging, depending on their coordinates, width and height - private boolean isCandidateForMerging(PdfImage image1, PdfImage image2) { - double x1 = image1.getPosition().getX(); - double y1 = image1.getPosition().getY(); - double width1 = image1.getPosition().getWidth(); - double x2 = image2.getPosition().getX(); - double y2 = image2.getPosition().getY(); - double width2 = image2.getPosition().getWidth(); - double height2 = image2.getPosition().getHeight(); - //if the x-coordinates and widths of images are equal and the height is equal to difference between y-coordinates, - // then it is the same picture and has to be merged -> return true - return x1 == x2 && width1 == width2 && Math.ceil(height2) == Math.ceil(y1 - y2) && width2 > (height2 / 6); - } - private void increaseDocumentStatistics(Page page, Document document) { @@ -319,5 +185,4 @@ public class PdfSegmentationService { } - } From 143877ff2d6003597cd2a8fc0fe8f835f9d96de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 16 Jun 2021 10:54:33 +0200 Subject: [PATCH 02/13] RED-1539: Fixed missing textBefore & textAfter on reanalyis that leads to wrong changelog --- .../redaction/v1/server/redaction/service/ReanalyzeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 3872f49f..9277d8b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -172,7 +172,7 @@ public class ReanalyzeService { Set entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection .getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false); - if (reanalysisSection.getCellStarts() != null) { + if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection .getCellStarts()); } else { From b166ab29ee16ab1bec7a8c310c6bd1cda238ae3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 17 Jun 2021 12:16:07 +0200 Subject: [PATCH 03/13] RED-1212: Enabled to exclude pages --- .../redaction/v1/model/AnalyzeRequest.java | 2 + .../v1/model/RedactionChangeLogEntry.java | 2 + .../redaction/v1/model/RedactionLogEntry.java | 2 + .../service/AnalyzeResponseService.java | 3 + .../redaction/service/ReanalyzeService.java | 75 ++++++++++++++----- .../service/RedactionChangeLogService.java | 1 + .../v1/server/RedactionIntegrationTest.java | 1 + 7 files changed, 68 insertions(+), 18 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 4aa290e9..1b7fa890 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -6,6 +6,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import java.time.OffsetDateTime; +import java.util.Set; @Data @Builder @@ -19,6 +20,7 @@ public class AnalyzeRequest { private boolean reanalyseOnlyIfPossible; private ManualRedactions manualRedactions; private OffsetDateTime lastProcessed; + private Set excludedPages; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java index 3dfbacce..a53d3b0e 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java @@ -44,4 +44,6 @@ public class RedactionChangeLogEntry { private boolean isDossierDictionaryEntry; + private boolean excluded; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index d8074d4f..f4c0dfd8 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -49,4 +49,6 @@ public class RedactionLogEntry { private boolean isDossierDictionaryEntry; + private boolean excluded; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index bae6d1d0..e8a3a885 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -14,16 +14,19 @@ public class AnalyzeResponseService { boolean hasRequests = redactionLog.getRedactionLogEntry() .stream() + .filter(entry -> !entry.isExcluded()) .anyMatch(entry -> entry.isManual() && entry.getStatus() .equals(com.iqser.red.service.redaction.v1.model.Status.REQUESTED)); boolean hasRedactions = redactionLog.getRedactionLogEntry() .stream() + .filter(entry -> !entry.isExcluded()) .anyMatch(entry -> entry.isRedacted() && !entry.isManual() || entry.isManual() && entry.getStatus() .equals(com.iqser.red.service.redaction.v1.model.Status.APPROVED)); boolean hasImages = redactionLog.getRedactionLogEntry() .stream() + .filter(entry -> !entry.isExcluded()) .anyMatch(entry -> entry.isHint() && entry.getType().equals("image")); boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 9277d8b9..9d3628e7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -1,14 +1,44 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.model.AnalyzeResult; +import com.iqser.red.service.redaction.v1.model.Comment; +import com.iqser.red.service.redaction.v1.model.IdRemoval; +import com.iqser.red.service.redaction.v1.model.ManualForceRedact; +import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; +import com.iqser.red.service.redaction.v1.model.ManualRedactions; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; +import com.iqser.red.service.redaction.v1.server.redaction.model.Section; +import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -17,14 +47,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestBody; - -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Slf4j @Service @RequiredArgsConstructor @@ -41,6 +63,7 @@ public class ReanalyzeService { private final AnalyzeResponseService analyzeResponseService; private final LegalBasisClient legalBasisClient; + public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { long startTime = System.currentTimeMillis(); @@ -66,11 +89,11 @@ public class ReanalyzeService { log.info("Redaction analysis successful..."); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis, - classifiedDoc.getDictionaryVersion().getDossierTemplateVersion(), - classifiedDoc.getDictionaryVersion().getDossierVersion(), - classifiedDoc.getRulesVersion(), - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() + .getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion() + .getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + + excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc .getDictionaryVersion(), analyzeRequest.getDossierTemplateId()); @@ -165,7 +188,8 @@ public class ReanalyzeService { KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest + .getDossierId()); List sectionSearchableTextPairs = new ArrayList<>(); for (SectionText reanalysisSection : reanalysisSections) { @@ -240,8 +264,7 @@ public class ReanalyzeService { .getDossierTemplateId())); } - redactionLog.getRedactionLogEntry() - .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); + redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); @@ -255,6 +278,8 @@ public class ReanalyzeService { redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierTemplateVersion()); redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion()); + excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); + var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); @@ -292,4 +317,18 @@ public class ReanalyzeService { .build(); } + + private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { + + redactionLog.getRedactionLogEntry().forEach(entry -> { + entry.getPositions().forEach(pos -> { + if (excludedPages != null && excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + } else { + entry.setExcluded(false); + } + }); + }); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index 53fc805e..1cc743e5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -90,6 +90,7 @@ public class RedactionChangeLogService { .comments(entry.getComments()) .changeType(changeType) .isDossierDictionaryEntry(entry.isDossierDictionaryEntry()) + .excluded(entry.isExcluded()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index a878400d..30d02971 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -633,6 +633,7 @@ public class RedactionIntegrationTest { long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf"); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); + request.setExcludedPages(Set.of(1)); AnalyzeResult result = reanalyzeService.analyze(request); From 8ff3e463eae960a35e11fce7589e24131ccb51f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Fri, 18 Jun 2021 11:28:43 +0200 Subject: [PATCH 04/13] RED-1326: Enabled to recategorize images --- .../v1/model/ManualImageRecategorization.java | 21 ++++++++++ .../v1/model/ManualRedactionType.java | 2 +- .../redaction/v1/model/ManualRedactions.java | 3 ++ .../redaction/v1/model/RedactionLogEntry.java | 2 + .../service/RedactionLogCreatorService.java | 40 +++++++++++++++++-- 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java new file mode 100644 index 00000000..7dc9120c --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ManualImageRecategorization { + + private String id; + private String user; + private Status status; + private String type; + private String legalBasis; + private boolean redacted; + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java index 83df7d67..31f78218 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.redaction.v1.model; public enum ManualRedactionType { - ADD, REMOVE, FORCE_REDACT + ADD, REMOVE, FORCE_REDACT, RECATEGORIZE } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java index af866d09..64426a07 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java @@ -26,6 +26,9 @@ public class ManualRedactions { @Builder.Default private Set entriesToAdd = new HashSet<>(); + @Builder.Default + private Set imageRecategorizations = new HashSet<>(); + @Builder.Default private Map> comments = new HashMap<>(); diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index f4c0dfd8..2c9e5610 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -51,4 +51,6 @@ public class RedactionLogEntry { private boolean excluded; + private String recategorizationType; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 9aef4dc7..92ac8a28 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -13,7 +13,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; + import lombok.RequiredArgsConstructor; + import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; @@ -87,6 +89,36 @@ public class RedactionLogCreatorService { .section(image.getSection()) .build(); + if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { + for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { + if (recategorization.getId().equals(id)) { + String manualOverrideReason = null; + if (recategorization.getStatus().equals(Status.APPROVED)) { + image.setType(recategorization.getType()); + image.setRedaction(recategorization.isRedacted()); + image.setLegalBasis(recategorization.getLegalBasis()); + redactionLogEntry.setRedacted(recategorization.isRedacted()); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setLegalBasis(recategorization.getLegalBasis()); + manualOverrideReason = image.getRedactionReason() + ", recategorized by manual override"; + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false)); + } else if (recategorization.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = image.getRedactionReason() + ", requested to recategorize"; + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); + redactionLogEntry.setRecategorizationType(recategorization.getType()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + } + } + } + if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { if (manualRemoval.getId().equals(id)) { @@ -276,20 +308,22 @@ public class RedactionLogCreatorService { List rectangles = new ArrayList<>(); if (textPositions.size() == 1) { - rectangles.add( TextPositionSequence.fromData(textPositions, page).getRectangle()); + rectangles.add(TextPositionSequence.fromData(textPositions, page).getRectangle()); } else { float y = textPositions.get(0).getYDirAdj(); int startIndex = 0; for (int i = 1; i < textPositions.size(); i++) { float yDirAdj = textPositions.get(i).getYDirAdj(); if (yDirAdj != y) { - rectangles.add( TextPositionSequence.fromData(textPositions.subList(startIndex, i), page).getRectangle()); + rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, i), page) + .getRectangle()); y = yDirAdj; startIndex = i; } } if (startIndex != textPositions.size()) { - rectangles.add( TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page).getRectangle()); + rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page) + .getRectangle()); } } From c5f5b9f7b928c31e25a000a6d9c055aab3e9fc00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Mon, 21 Jun 2021 15:10:21 +0200 Subject: [PATCH 05/13] RED-1326: Fixed image recatigorization in reanalysis --- .../redaction/service/ReanalyzeService.java | 6 +++--- .../service/RedactionLogCreatorService.java | 2 ++ .../v1/server/RedactionIntegrationTest.java | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 9d3628e7..17fed969 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.Comment; import com.iqser.red.service.redaction.v1.model.IdRemoval; import com.iqser.red.service.redaction.v1.model.ManualForceRedact; +import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.Rectangle; @@ -296,9 +297,8 @@ public class ReanalyzeService { return new HashSet<>(); } - return Stream.concat(manualRedactions.getIdsToRemove() - .stream() - .map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)) + return Stream.concat(manualRedactions.getImageRecategorizations().stream().map(ManualImageRecategorization::getId), + Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 92ac8a28..6a6a33e0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -97,6 +97,8 @@ public class RedactionLogCreatorService { image.setType(recategorization.getType()); image.setRedaction(recategorization.isRedacted()); image.setLegalBasis(recategorization.getLegalBasis()); + redactionLogEntry.setType(recategorization.getType()); + redactionLogEntry.setHint(dictionaryService.isHint(recategorization.getType(), dossierTemplateId)); redactionLogEntry.setRedacted(recategorization.isRedacted()); redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setLegalBasis(recategorization.getLegalBasis()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 30d02971..ab810467 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -689,8 +689,25 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false)); start = System.currentTimeMillis(); + + ManualRedactions manualRedactions = new ManualRedactions(); + + manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder() + .id("37eee3e9d589a5cc529bfec38c3ba479") + .status(Status.APPROVED) + .type("signature") + .redacted(true) + .legalBasis("Article 39(e)(1) and Article 39(e)(2) of Regulation (EC) No 178/2002") + .build())); + + request.setManualRedactions(manualRedactions); + + AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + end = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (end - start)); From 7c99581f32fa4ac963cf0bd32bbc0dfe05903d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 22 Jun 2021 12:09:35 +0200 Subject: [PATCH 06/13] RED-1446: Enabled to change legal basis manually --- .../v1/model/ManualLegalBasisChange.java | 19 +++++++ .../v1/model/ManualRedactionType.java | 2 +- .../redaction/v1/model/ManualRedactions.java | 3 ++ .../redaction/v1/model/RedactionLogEntry.java | 1 + .../redaction/service/ReanalyzeService.java | 6 ++- .../service/RedactionLogCreatorService.java | 51 +++++++++++++++++++ .../v1/server/RedactionIntegrationTest.java | 8 +++ 7 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java new file mode 100644 index 00000000..4f0d211f --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ManualLegalBasisChange { + + private String id; + private String user; + private Status status; + private String legalBasis; + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java index 31f78218..0bb0c607 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.redaction.v1.model; public enum ManualRedactionType { - ADD, REMOVE, FORCE_REDACT, RECATEGORIZE + ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java index 64426a07..baffede0 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java @@ -29,6 +29,9 @@ public class ManualRedactions { @Builder.Default private Set imageRecategorizations = new HashSet<>(); + @Builder.Default + private Set manualLegalBasisChanges = new HashSet<>(); + @Builder.Default private Map> comments = new HashMap<>(); diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 2c9e5610..e347f0b3 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -52,5 +52,6 @@ public class RedactionLogEntry { private boolean excluded; private String recategorizationType; + private String legalBasisChangeValue; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 17fed969..4f20e29f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -20,6 +20,7 @@ import com.iqser.red.service.redaction.v1.model.Comment; import com.iqser.red.service.redaction.v1.model.IdRemoval; import com.iqser.red.service.redaction.v1.model.ManualForceRedact; import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; +import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.Rectangle; @@ -297,8 +298,9 @@ public class ReanalyzeService { return new HashSet<>(); } - return Stream.concat(manualRedactions.getImageRecategorizations().stream().map(ManualImageRecategorization::getId), - Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))) + return Stream.concat(manualRedactions.getManualLegalBasisChanges().stream().map(ManualLegalBasisChange::getId), + Stream.concat(manualRedactions.getImageRecategorizations().stream().map(ManualImageRecategorization::getId), + Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 6a6a33e0..ba094a48 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -175,6 +175,32 @@ public class RedactionLogCreatorService { } } + + if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { + for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { + if (manualLegalBasisChange.getId().equals(id)) { + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = image.getRedactionReason() + ", legal basis was manually changed"; + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = image.getRedactionReason() + ", legal basis change requested"; + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + } + } + redactionLogEntities.add(redactionLogEntry); } @@ -278,6 +304,31 @@ public class RedactionLogCreatorService { } } + if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { + for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { + if (manualLegalBasisChange.getId().equals(entityPositionSequence.getId())) { + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = entity.getRedactionReason() + ", legal basis was manually changed"; + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = entity.getRedactionReason() + ", legal basis change requested"; + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true)); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + } + } + if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { List rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences() .stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index ab810467..57cfb128 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -775,6 +775,7 @@ public class RedactionIntegrationTest { .status(Status.APPROVED) .build())); + manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment)); manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment)); manualRedactions.getComments().put(manualAddId, List.of(comment)); @@ -794,14 +795,21 @@ public class RedactionIntegrationTest { request.setManualRedactions(manualRedactions); AnalyzeResult result = reanalyzeService.analyze(request); + manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() .id("5b940b2cb401ed9f5be6fc24f6e77bcf") .status(Status.APPROVED) .build())); + manualRedactions.setManualLegalBasisChanges(Set.of(ManualLegalBasisChange.builder() + .id("675eba69b0c2917de55462c817adaa05") + .legalBasis("Manual Legal Basis Change") + .status(Status.APPROVED) + .build())); reanalyzeService.reanalyze(request); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() .dossierId(TEST_DOSSIER_ID) From 5d5b3529cd8e45493cc91df3cb8b1f2a3ba14b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Fri, 25 Jun 2021 10:57:20 +0200 Subject: [PATCH 07/13] RED-1637: Fixed hasImages flag --- .../v1/server/redaction/service/AnalyzeResponseService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index e8a3a885..eceedae2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -27,7 +27,7 @@ public class AnalyzeResponseService { boolean hasImages = redactionLog.getRedactionLogEntry() .stream() .filter(entry -> !entry.isExcluded()) - .anyMatch(entry -> entry.isHint() && entry.getType().equals("image")); + .anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage()); boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog .getRedactionLogEntry() From 6ae6d467fc5fec4b385cde1c70bae7a21143342f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Tue, 29 Jun 2021 10:24:51 +0200 Subject: [PATCH 08/13] RED-1445: Enabled to use file attributes in rules --- .../redaction/v1/model/AnalyzeRequest.java | 7 +++++ .../redaction/v1/model/FileAttribute.java | 19 ++++++++++++ .../v1/server/redaction/model/Section.java | 19 ++++++++++++ .../service/EntityRedactionService.java | 31 +++++++++++-------- .../redaction/service/ReanalyzeService.java | 3 +- .../v1/server/RedactionIntegrationTest.java | 2 ++ .../service/EntityRedactionServiceTest.java | 20 ++++++------ .../src/test/resources/drools/rules.drl | 2 +- 8 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 1b7fa890..7a891277 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -6,6 +6,10 @@ import lombok.Data; import lombok.NoArgsConstructor; import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Set; @Data @@ -22,5 +26,8 @@ public class AnalyzeRequest { private OffsetDateTime lastProcessed; private Set excludedPages; + @Builder.Default + private List fileAttributes = new ArrayList<>(); + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java new file mode 100644 index 00000000..aea494c9 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FileAttribute { + + private String id; + private String label; + private String placeholder; + private String value; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index fa44f983..dc878a20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; @@ -8,9 +9,11 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -52,6 +55,22 @@ public class Section { @Builder.Default private Set images = new HashSet<>(); + @Builder.Default + private List fileAttributes = new ArrayList<>(); + + + public boolean fileAttributeByIdEquals(String id, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue())).findFirst().isPresent(); + } + + public boolean fileAttributeByPlaceholderEquals(String placeholder, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue())).findFirst().isPresent(); + } + + public boolean fileAttributeByLabelEquals(String label, String value){ + return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())).findFirst().isPresent(); + } + public boolean rowEquals(String headerName, String value) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 02da3aa7..e6d00c15 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.Point; @@ -33,7 +34,7 @@ public class EntityRedactionService { private final SurroundingWordsService surroundingWordsService; - public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId) { + public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId, List fileAttributes) { dictionaryService.updateDictionary(dossierTemplateId, dossierId); KieContainer container = droolsExecutionService.updateRules(dossierTemplateId); @@ -41,7 +42,7 @@ public class EntityRedactionService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(dossierTemplateId, dossierId); - Set documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null)); + Set documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null, fileAttributes)); if (dictionary.hasLocalEntries()) { @@ -53,7 +54,7 @@ public class EntityRedactionService { } }); - Set foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber); + Set foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber, fileAttributes); EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary); EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities); } @@ -84,7 +85,7 @@ public class EntityRedactionService { private Set findEntities(Document classifiedDoc, KieContainer kieContainer, ManualRedactions manualRedactions, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber) { + Map> hintsPerSectionNumber, List fileAttributes) { Set documentEntities = new HashSet<>(); @@ -95,31 +96,31 @@ public class EntityRedactionService { List tables = paragraph.getTables(); for (Table table : tables) { if (table.getColCount() == 2) { - sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } else { - sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } sectionNumber.incrementAndGet(); } sectionSearchableTextPairs.add(processText(classifiedDoc, paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph .getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, paragraph - .getImages())); + .getImages(), fileAttributes)); sectionNumber.incrementAndGet(); } for (Header header : classifiedDoc.getHeaders()) { - sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>())); + sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>(), fileAttributes)); sectionNumber.incrementAndGet(); } for (Footer footer : classifiedDoc.getFooters()) { - sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>())); + sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>(), fileAttributes)); sectionNumber.incrementAndGet(); } for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) { sectionSearchableTextPairs.add(processText(classifiedDoc, unclassifiedText.getSearchableText(), unclassifiedText - .getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>())); + .getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>(), fileAttributes)); sectionNumber.incrementAndGet(); } @@ -164,7 +165,7 @@ public class EntityRedactionService { ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber) { + Map> hintsPerSectionNumber, List fileAttributes) { List sectionSearchableTextPairs = new ArrayList<>(); @@ -229,6 +230,7 @@ public class EntityRedactionService { .tabularData(tabularData) .searchableText(searchableRow) .dictionary(dictionary) + .fileAttributes(fileAttributes) .build(), searchableRow)); if (!local) { @@ -252,7 +254,8 @@ public class EntityRedactionService { ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, - Map> hintsPerSectionNumber) { + Map> hintsPerSectionNumber, + List fileAttributes) { List sectionSearchableTextPairs = new ArrayList<>(); SearchableText entireTableText = new SearchableText(); @@ -296,6 +299,7 @@ public class EntityRedactionService { .sectionNumber(sectionNumber.intValue()) .searchableText(entireTableText) .dictionary(dictionary) + .fileAttributes(fileAttributes) .build(), entireTableText)); if (!local) { @@ -315,7 +319,7 @@ public class EntityRedactionService { ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, - List images) { + List images, List fileAttributes) { if (!local) { SectionText sectionText = new SectionText(); @@ -355,6 +359,7 @@ public class EntityRedactionService { .images(images.stream() .map(image -> convert(image, sectionNumber.intValue(), headline)) .collect(Collectors.toSet())) + .fileAttributes(fileAttributes) .build(), searchableText); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 4f20e29f..edaac63d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -84,7 +84,7 @@ public class ReanalyzeService { log.info("Document structure analysis successful, starting redaction analysis..."); entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest - .getDossierId()); + .getDossierId(), analyzeRequest.getFileAttributes()); redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest .getDossierTemplateId()); @@ -217,6 +217,7 @@ public class ReanalyzeService { .searchableText(reanalysisSection.getSearchableText()) .dictionary(dictionary) .images(reanalysisSection.getImages()) + .fileAttributes(analyzeRequest.getFileAttributes()) .build(), reanalysisSection.getSearchableText())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 57cfb128..305df362 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -635,6 +635,8 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setExcludedPages(Set.of(1)); + request.setFileAttributes(List.of(FileAttribute.builder().id("fileAttributeId").label("Vertebrate Study").placeholder("{fileattributes.vertebrateStudy}").value("true").build())); + AnalyzeResult result = reanalyzeService.analyze(request); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java index 01379d9b..1ab88196 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java @@ -151,7 +151,7 @@ public class EntityRedactionServiceTest { when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities } @@ -177,7 +177,7 @@ public class EntityRedactionServiceTest { when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities } @@ -202,7 +202,7 @@ public class EntityRedactionServiceTest { .build(); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities() .entrySet() .stream() @@ -210,7 +210,7 @@ public class EntityRedactionServiceTest { pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " + "the plant protection product.pdf"); classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities() .entrySet() .stream() @@ -235,7 +235,7 @@ public class EntityRedactionServiceTest { .build(); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream() .filter(entity -> entity.getMatchedRule() == 9) @@ -302,7 +302,7 @@ public class EntityRedactionServiceTest { .build(); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream() .filter(entity -> entity.getMatchedRule() == 6) @@ -341,7 +341,7 @@ public class EntityRedactionServiceTest { .build(); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(dictionaryResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream() .filter(entity -> entity.getMatchedRule() == 11) @@ -371,7 +371,7 @@ public class EntityRedactionServiceTest { .build(); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8); assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y @@ -390,7 +390,7 @@ public class EntityRedactionServiceTest { when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(addressResponse); classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3); assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9); @@ -419,7 +419,7 @@ public class EntityRedactionServiceTest { .build(); when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(sponsorResponse); Document classifiedDoc = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream()); - entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId"); + entityRedactionService.processDocument(classifiedDoc, TEST_DOSSIER_TEMPLATE_ID, null, "dossierId", null); assertThat(classifiedDoc.getEntities()).hasSize(1); // one page assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 5f7e24f2..853d7fac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -268,7 +268,7 @@ rule "18: Redact contact information if Producer is found" rule "19: Redact AUTHOR(S)" when - Section(searchText.contains("AUTHOR(S):")) + Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true")) then section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); end From 24ed2498ea1be2fc4a4daeda81c9a5371e8ad7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 1 Jul 2021 09:16:32 +0200 Subject: [PATCH 09/13] RED-1728: Do not show isHint flag if only false_positive entries are included in redactionlog --- .../v1/server/redaction/service/AnalyzeResponseService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index eceedae2..80589a8f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -10,7 +10,7 @@ import org.springframework.stereotype.Service; public class AnalyzeResponseService { public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { - boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(RedactionLogEntry::isHint); + boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.isHint() && !entry.getType().equals("false_positive")); boolean hasRequests = redactionLog.getRedactionLogEntry() .stream() From 041a3c87ae908fb8e127d4719aa421cbf6c3621e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Mon, 5 Jul 2021 14:28:53 +0200 Subject: [PATCH 10/13] RED-809: Added flag to AnalysisResult to see if a reanalysis or normal analysis was performed --- .../iqser/red/service/redaction/v1/model/AnalyzeResult.java | 2 ++ .../v1/server/redaction/service/ReanalyzeService.java | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index 1ce9d759..5eadb70f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -25,6 +25,8 @@ public class AnalyzeResult { private long rulesVersion; private long legalBasisVersion; + private boolean wasReanalyzed; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index edaac63d..ddc67900 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -269,8 +269,9 @@ public class ReanalyzeService { redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); - + AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); + analyzeResult.setWasReanalyzed(true); + return analyzeResult; } From c36a3813c42d37fada16d39479d4205d4890a7b5 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 7 Jul 2021 23:05:49 +0300 Subject: [PATCH 11/13] added a preview method that allows us to see how the redaction-log will look like after analysis, this is useful for the UI since we do no longer need to duplicate this code in the UI --- .../v1/resources/RedactionResource.java | 5 +- .../controller/RedactionController.java | 12 +- .../v1/server/redaction/model/Entity.java | 2 +- .../v1/server/redaction/model/Image.java | 2 +- .../server/redaction/model/ReasonHolder.java | 14 + .../service/RedactionLogCreatorService.java | 522 ++++++++++-------- 6 files changed, 335 insertions(+), 222 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ReasonHolder.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index cdf45fcb..0a6037a6 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -8,8 +8,6 @@ import org.springframework.web.bind.annotation.RequestBody; public interface RedactionResource { - String SERVICE_NAME = "redaction-service-v1"; - String RULE_SET_PARAMETER_NAME = "dossierTemplateId"; String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}"; @@ -32,4 +30,7 @@ public interface RedactionResource { @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) void testRules(@RequestBody String rules); + @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE) + RedactionLog getRedactionLogPreview(@RequestBody RedactionRequest redactionRequest); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index a2e285e4..0149d6a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.controller; import com.iqser.red.service.file.management.v1.api.model.FileType; import com.iqser.red.service.redaction.v1.model.AnnotateRequest; import com.iqser.red.service.redaction.v1.model.AnnotateResponse; +import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionRequest; import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.service.redaction.v1.resources.RedactionResource; @@ -12,6 +13,7 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; @@ -39,7 +41,7 @@ public class RedactionController implements RedactionResource { private final AnnotationService annotationService; private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; - + private final RedactionLogCreatorService redactionLogCreatorService; public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { @@ -155,6 +157,14 @@ public class RedactionController implements RedactionResource { droolsExecutionService.testRules(rules); } + @Override + public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) { + + var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); + + return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + } + private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index c9fdc711..2ae553db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -9,7 +9,7 @@ import java.util.List; @Data @EqualsAndHashCode(onlyExplicitlyIncluded = true) -public class Entity { +public class Entity implements ReasonHolder { private final String word; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java index 766d607d..5aab9c7a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java @@ -9,7 +9,7 @@ import lombok.NoArgsConstructor; @Builder @NoArgsConstructor @AllArgsConstructor -public class Image { +public class Image implements ReasonHolder { private String type; private RedRectangle2D position; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ReasonHolder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ReasonHolder.java new file mode 100644 index 00000000..51d9b0fd --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ReasonHolder.java @@ -0,0 +1,14 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +public interface ReasonHolder { + + String getRedactionReason(); + + void setRedactionReason(String reason); + + boolean isRedaction(); + + void setRedaction(boolean value); + + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index ba094a48..135d90a2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -9,21 +9,17 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.ReasonHolder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; - import lombok.RequiredArgsConstructor; - import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Service @@ -71,7 +67,7 @@ public class RedactionLogCreatorService { RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() .id(id) - .color(getColorForImage(image, dossierTemplateId, false)) + .color(getColorForImage(image.getType(), dossierTemplateId, false, image.isRedaction())) .isImage(true) .type(image.getType()) .redacted(image.isRedaction()) @@ -89,117 +85,7 @@ public class RedactionLogCreatorService { .section(image.getSection()) .build(); - if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { - for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { - if (recategorization.getId().equals(id)) { - String manualOverrideReason = null; - if (recategorization.getStatus().equals(Status.APPROVED)) { - image.setType(recategorization.getType()); - image.setRedaction(recategorization.isRedacted()); - image.setLegalBasis(recategorization.getLegalBasis()); - redactionLogEntry.setType(recategorization.getType()); - redactionLogEntry.setHint(dictionaryService.isHint(recategorization.getType(), dossierTemplateId)); - redactionLogEntry.setRedacted(recategorization.isRedacted()); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setLegalBasis(recategorization.getLegalBasis()); - manualOverrideReason = image.getRedactionReason() + ", recategorized by manual override"; - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false)); - } else if (recategorization.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = image.getRedactionReason() + ", requested to recategorize"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); - redactionLogEntry.setRecategorizationType(recategorization.getType()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(id)) { - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - image.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = image.getRedactionReason() + ", removed by manual override"; - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false)); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = image.getRedactionReason() + ", requested to remove"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(id)) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - image.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false)); - manualOverrideReason = image.getRedactionReason() + ", forced by manual override"; - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = image.getRedactionReason() + ", requested to force redact"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - } - } - } - - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(id)) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = image.getRedactionReason() + ", legal basis was manually changed"; - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = image.getRedactionReason() + ", legal basis change requested"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true)); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } + processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry); redactionLogEntities.add(redactionLogEntry); } @@ -207,6 +93,118 @@ public class RedactionLogCreatorService { return redactionLogEntities; } + private void processImageEntry(ManualRedactions manualRedactions, String dossierTemplateId, ReasonHolder image, RedactionLogEntry redactionLogEntry) { + if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { + for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { + if (recategorization.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (recategorization.getStatus().equals(Status.APPROVED)) { + image.setRedaction(recategorization.isRedacted()); + redactionLogEntry.setType(recategorization.getType()); + redactionLogEntry.setHint(dictionaryService.isHint(recategorization.getType(), dossierTemplateId)); + redactionLogEntry.setRedacted(recategorization.isRedacted()); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setLegalBasis(recategorization.getLegalBasis()); + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override"); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); + } else if (recategorization.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + redactionLogEntry.setRecategorizationType(recategorization.getType()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + } + } + } + + if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { + for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { + if (manualRemoval.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualRemoval.getStatus().equals(Status.APPROVED)) { + image.setRedaction(false); + redactionLogEntry.setRedacted(false); + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", removed by manual override"); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); + } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to remove"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + } + } + } + + if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { + for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { + if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + image.setRedaction(true); + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", forced by manual override"); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to force redact"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + } + } + } + + + if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { + for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { + if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis was manually changed"); + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis change requested"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + } + } + } + private Set getManualRedactionPages(ManualRedactions manualRedactions) { @@ -236,7 +234,6 @@ public class RedactionLogCreatorService { entityLoop: for (Entity entity : entities.get(page)) { - List comments = null; for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { @@ -249,85 +246,8 @@ public class RedactionLogCreatorService { processedIds.add(entityPositionSequence.getId()); } - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(entityPositionSequence.getId())) { - comments = manualRedactions.getComments().get(manualRemoval.getId()); - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - entity.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = entity.getRedactionReason() + ", removed by manual override"; - redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false)); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = entity.getRedactionReason() + ", requested to remove"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true)); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(entityPositionSequence.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - entity.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false)); - manualOverrideReason = entity.getRedactionReason() + ", forced by manual override"; - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = entity.getRedactionReason() + ", requested to force redact"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true)); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - } - } - } - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(entityPositionSequence.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = entity.getRedactionReason() + ", legal basis was manually changed"; - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = entity.getRedactionReason() + ", legal basis change requested"; - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true)); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } + redactionLogEntry.setId(entityPositionSequence.getId()); + processRedactionLogEntry(manualRedactions, dossierTemplateId, redactionLogEntry, entity); if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { List rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences() @@ -335,16 +255,11 @@ public class RedactionLogCreatorService { .flatMap(seq -> seq.getTextPositions().stream()) .collect(Collectors.toList()), page); - if (manualRedactions != null) { - comments = manualRedactions.getComments().get(entityPositionSequence.getId()); - } - redactionLogEntry.setComments(comments); redactionLogEntry.getPositions().addAll(rectanglesPerLine); } - redactionLogEntry.setId(entityPositionSequence.getId()); // FIXME ids should never be null. Figure out why this happens. if (redactionLogEntry.getId() != null) { @@ -356,6 +271,109 @@ public class RedactionLogCreatorService { return redactionLogEntities; } + private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry, ReasonHolder reasonHolder) { + + List comments = null; + + if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { + for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { + if (manualRemoval.getId().equals(redactionLogEntry.getId())) { + comments = manualRedactions.getComments().get(manualRemoval.getId()); + String manualOverrideReason = null; + if (manualRemoval.getStatus().equals(Status.APPROVED)) { + reasonHolder.setRedaction(false); + redactionLogEntry.setRedacted(false); + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", removed by manual override"); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); + } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to remove"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + } + } + } + + if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { + for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { + if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + reasonHolder.setRedaction(true); + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); + manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", forced by manual override"); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to force redact"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + } + } + } + + if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { + for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { + if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis was manually changed"); + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis change requested"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + } + } + + + if (manualRedactions != null) { + comments = manualRedactions.getComments().get(redactionLogEntry.getId()); + } + + redactionLogEntry.setComments(comments); + } + + private String mergeReasonIfNecessary(String currentReason, String addition) { + if (currentReason != null) { + if (!currentReason.contains(addition)) { + currentReason += addition; + } + return currentReason; + } else { + return ""; + } + } + private List getRectanglesPerLine(List textPositions, int page) { @@ -450,14 +468,14 @@ public class RedactionLogCreatorService { private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { return RedactionLogEntry.builder() - .color(getColor(entity, dossierTemplateId, false)) + .color(getColor(entity.getType(), dossierTemplateId, false, entity.isRedaction())) .reason(entity.getRedactionReason()) .legalBasis(entity.getLegalBasis()) .value(entity.getWord()) .type(entity.getType()) .redacted(entity.isRedaction()) - .isHint(isHint(entity, dossierTemplateId)) - .isRecommendation(isRecommendation(entity, dossierTemplateId)) + .isHint(isHint(entity.getType(), dossierTemplateId)) + .isRecommendation(isRecommendation(entity.getType(), dossierTemplateId)) .section(entity.getHeadline()) .sectionNumber(entity.getSectionNumber()) .matchedRule(entity.getMatchedRule()) @@ -471,15 +489,15 @@ public class RedactionLogCreatorService { } - private float[] getColor(Entity entity, String dossierTemplateId, boolean requestedToRemove) { + private float[] getColor(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { if (requestedToRemove) { return dictionaryService.getRequestRemoveColor(dossierTemplateId); } - if (!entity.isRedaction() && !isHint(entity, dossierTemplateId)) { + if (!isRedaction && !isHint(type, dossierTemplateId)) { return dictionaryService.getNotRedactedColor(dossierTemplateId); } - return dictionaryService.getColor(entity.getType(), dossierTemplateId); + return dictionaryService.getColor(type, dossierTemplateId); } @@ -500,27 +518,27 @@ public class RedactionLogCreatorService { } - private float[] getColorForImage(Image image, String dossierTemplateId, boolean requestedToRemove) { + private float[] getColorForImage(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { if (requestedToRemove) { return dictionaryService.getRequestRemoveColor(dossierTemplateId); } - if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), dossierTemplateId)) { + if (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) { return dictionaryService.getNotRedactedColor(dossierTemplateId); } - return dictionaryService.getColor(image.getType(), dossierTemplateId); + return dictionaryService.getColor(type, dossierTemplateId); } - private boolean isHint(Entity entity, String dossierTemplateId) { + private boolean isHint(String type, String dossierTemplateId) { - return dictionaryService.isHint(entity.getType(), dossierTemplateId); + return dictionaryService.isHint(type, dossierTemplateId); } - private boolean isRecommendation(Entity entity, String dossierTemplateId) { + private boolean isRecommendation(String type, String dossierTemplateId) { - return dictionaryService.isRecommendation(entity.getType(), dossierTemplateId); + return dictionaryService.isRecommendation(type, dossierTemplateId); } @@ -567,4 +585,74 @@ public class RedactionLogCreatorService { } } + public RedactionLog getRedactionLogPreview(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + + + var manualRedactionPages = getManualRedactionPages(manualRedactions); + + // generate all manual entries + var manualRedactionLogEntries = new HashMap(); + for (var page : manualRedactionPages) { + + var pageEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId); + + for (var entry : pageEntries) { + manualRedactionLogEntries.put(entry.getId(), entry); + } + } + + for (var manualEntry : manualRedactionLogEntries.values()) { + var existingEntry = redactionLog.getRedactionLogEntry().stream().filter(e -> e.getId().equals(manualEntry.getId())).findAny(); + if (existingEntry.isPresent()) { + // if it has already been processed of sorts, update it + BeanUtils.copyProperties(manualEntry, existingEntry.get()); + } else { + // not yet in the redaction-log - add it + redactionLog.getRedactionLogEntry().add(manualEntry); + } + } + + + for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { + + var reasonHolder = new PreviewReasonHolder(entry); + + if (entry.isImage()) { + processImageEntry(manualRedactions, dossierTemplateId, reasonHolder, entry); + } + + processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder); + } + + return redactionLog; + } + + public static class PreviewReasonHolder implements ReasonHolder { + + private final RedactionLogEntry entry; + + public PreviewReasonHolder(RedactionLogEntry entry) { + this.entry = entry; + } + + @Override + public String getRedactionReason() { + return entry.getReason(); + } + + @Override + public void setRedactionReason(String reason) { + entry.setReason(reason); + } + + @Override + public boolean isRedaction() { + return entry.isRedacted(); + } + + @Override + public void setRedaction(boolean value) { + entry.setRedacted(value); + } + } } From 9225b9959ded9768c32ed5e23b50e2428117952f Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 7 Jul 2021 23:17:03 +0300 Subject: [PATCH 12/13] updated platform dep version and fixed pmd issue --- redaction-service-v1/pom.xml | 4 ++-- .../server/redaction/service/RedactionLogCreatorService.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/pom.xml b/redaction-service-v1/pom.xml index ee796600..eddd4fbb 100644 --- a/redaction-service-v1/pom.xml +++ b/redaction-service-v1/pom.xml @@ -5,7 +5,7 @@ platform-dependency com.iqser.red - 1.1.2 + 1.1.3 4.0.0 @@ -32,7 +32,7 @@ com.iqser.red platform-commons-dependency - 1.3.1 + 1.3.6 import pom diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 135d90a2..5c5e1e70 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -366,7 +366,7 @@ public class RedactionLogCreatorService { private String mergeReasonIfNecessary(String currentReason, String addition) { if (currentReason != null) { if (!currentReason.contains(addition)) { - currentReason += addition; + return currentReason + addition; } return currentReason; } else { From abb48d46f95ab218c790dece094a44b8e9ccf86c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 8 Jul 2021 11:31:27 +0200 Subject: [PATCH 13/13] RED-1755: Fixed hasHints flag at excluded pages --- .../redaction/service/AnalyzeResponseService.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index 80589a8f..ff772521 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -4,13 +4,19 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; + import org.springframework.stereotype.Service; @Service public class AnalyzeResponseService { - public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { - boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.isHint() && !entry.getType().equals("false_positive")); + public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, + RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { + + boolean hasHints = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> !entry.isExcluded()) + .anyMatch(entry -> entry.isHint() && !entry.getType().equals("false_positive")); boolean hasRequests = redactionLog.getRedactionLogEntry() .stream() @@ -31,7 +37,9 @@ public class AnalyzeResponseService { boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog .getRedactionLogEntry() - .isEmpty() && redactionChangeLog.getRedactionLogEntry().stream().anyMatch(entry -> !entry.getType().equals("false_positive")); + .isEmpty() && redactionChangeLog.getRedactionLogEntry() + .stream() + .anyMatch(entry -> !entry.getType().equals("false_positive")); return AnalyzeResult.builder() .dossierId(dossierId) @@ -49,4 +57,5 @@ public class AnalyzeResponseService { .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion()) .build(); } + }