Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'master' of ssh://git.iqser.com:2222/red/redaction-service into Test

Browse Source
This commit is contained in:
aoezyetimoglu 2021-08-27 14:32:32 +02:00
commit 6e9724f5bc
46 changed files with 1061 additions and 959 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-image-v1/src/main/docker/Dockerfile
View File

@ -13,6 +13,6 @@ RUN apt-get update \
wget cabextract xfonts-utils fonts-liberation \
&& rm -rf /var/lib/apt/lists/*
RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.7_all.deb -o /tmp/ttf-mscorefonts-installer_3.7_all.deb \
&& dpkg -i /tmp/ttf-mscorefonts-installer_3.7_all.deb \
&& rm /tmp/ttf-mscorefonts-installer_3.7_all.deb \
RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb -o /tmp/ttf-mscorefonts-installer_3.8_all.deb \
&& dpkg -i /tmp/ttf-mscorefonts-installer_3.8_all.deb \
&& rm /tmp/ttf-mscorefonts-installer_3.8_all.deb \

9
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -7,9 +7,8 @@ import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Data
@ -24,7 +23,11 @@ public class AnalyzeRequest {
private boolean reanalyseOnlyIfPossible;
private ManualRedactions manualRedactions;
private OffsetDateTime lastProcessed;
private Set<Integer> excludedPages;
@Builder.Default
private Set<Integer> excludedPages = new HashSet<>();
@Builder.Default
private Set<Integer> sectionsToReanalyse = new HashSet<>();
@Builder.Default
private List<FileAttribute> fileAttributes = new ArrayList<>();

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java
View File

@ -15,10 +15,6 @@ public class AnalyzeResult {
private String fileId;
private long duration;
private int numberOfPages;
private boolean hasHints;
private boolean hasRequests;
private boolean hasRedactions;
private boolean hasImages;
private boolean hasUpdates;
private long dictionaryVersion;
private long dossierDictionaryVersion;
@ -27,6 +23,8 @@ public class AnalyzeResult {
private boolean wasReanalyzed;
private int analysisVersion;
}

18
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java Normal file
View File

@ -0,0 +1,18 @@
package com.iqser.red.service.redaction.v1.model;
import java.time.OffsetDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Change {
private ChangeType type;
private OffsetDateTime dateTime;
}

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java
View File

@ -1,5 +1,5 @@
package com.iqser.red.service.redaction.v1.model;
public enum ChangeType {
ADDED, REMOVED
ADDED, REMOVED, CHANGED
}

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java
View File

@ -5,6 +5,8 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@ -16,4 +18,8 @@ public class IdRemoval {
private Status status;
private boolean removeFromDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java
View File

@ -5,6 +5,8 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@ -16,4 +18,8 @@ public class ManualForceRedact {
private Status status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

8
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java
View File

@ -5,6 +5,8 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@ -15,7 +17,9 @@ public class ManualImageRecategorization {
private String user;
private Status status;
private String type;
private String legalBasis;
private boolean redacted;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java
View File

@ -5,6 +5,8 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@ -16,4 +18,8 @@ public class ManualLegalBasisChange {
private Status status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

10
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java
View File

@ -5,6 +5,7 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
@ -22,11 +23,12 @@ public class ManualRedactionEntry {
private String legalBasis;
private List<Rectangle> positions = new ArrayList<>();
private Status status;
private boolean addToDictionary;
private String section;
private int sectionNumber;
private boolean addToDossierDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

22
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java
View File

@ -1,22 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class RedactionChangeLog {
private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
}

49
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java
View File

@ -1,49 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RedactionChangeLogEntry {
private String id;
private String type;
private String value;
private String reason;
private int matchedRule;
private String legalBasis;
private boolean redacted;
private boolean isHint;
private boolean isRecommendation;
private String section;
private float[] color;
@Builder.Default
private List<Rectangle> positions = new ArrayList<>();
private int sectionNumber;
private boolean manual;
private Status status;
private ManualRedactionType manualRedactionType;
private boolean isDictionaryEntry;
private String textBefore;
private String textAfter;
@Builder.Default
private List<Comment> comments = new ArrayList<>();
private ChangeType changeType;
private boolean isDossierDictionaryEntry;
private boolean excluded;
}

7
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java
View File

@ -10,6 +10,13 @@ import java.util.List;
@AllArgsConstructor
public class RedactionLog {
/**
* Version 0 Redaction Logs have manual redactions merged inside them
* Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
*/
private long analysisVersion;
private List<RedactionLogEntry> redactionLogEntry;
private List<LegalBasisMapping> legalBasis;

17
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java Normal file
View File

@ -0,0 +1,17 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class RedactionLogChanges {
private RedactionLog redactionLog;
private boolean hasChanges;
}

8
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -9,6 +9,8 @@ import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@ -34,6 +36,7 @@ public class RedactionLogEntry {
private boolean manual;
private Status status;
private ManualRedactionType manualRedactionType;
private String manualRedactionUserId;
private boolean isDictionaryEntry;
private String textBefore;
@ -46,6 +49,7 @@ public class RedactionLogEntry {
private int endOffset;
private boolean isImage;
private boolean imageHasTransparency;
private boolean isDossierDictionaryEntry;
@ -54,4 +58,8 @@ public class RedactionLogEntry {
private String recategorizationType;
private String legalBasisChangeValue;
@EqualsAndHashCode.Exclude
@Builder.Default
private List<Change> changes = new ArrayList<>();
}

5
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -1,5 +1,8 @@
package com.iqser.red.service.redaction.v1.model;
import java.util.HashSet;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -15,4 +18,6 @@ public class RedactionRequest {
private String fileId;
private String dossierTemplateId;
private ManualRedactions manualRedactions;
@Builder.Default
private Set<Integer> excludedPages = new HashSet<>();
}

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java
View File

@ -30,4 +30,5 @@ public class SectionRectangle {
private int numberOfParts;
private List<CellRectangle> tableCells;
}

9
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java
View File

@ -8,10 +8,6 @@ import org.springframework.web.bind.annotation.RequestBody;
public interface RedactionResource {
String RULE_SET_PARAMETER_NAME = "dossierTemplateId";
String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";
@PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
@ -24,13 +20,10 @@ public interface RedactionResource {
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
@PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId);
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
void testRules(@RequestBody String rules);
@PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE)
RedactionLog getRedactionLogPreview(@RequestBody RedactionRequest redactionRequest);
RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest);
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java Normal file
View File

@ -0,0 +1,18 @@
package com.iqser.red.service.redaction.v1.server.client;
import java.util.List;
import java.util.Map;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}")
public interface EntityRecognitionClient {
@PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
Map<String, Map<String, List<EntityRecogintionEntity>>> findAuthors(EntityRecognitionRequest entityRecognitionRequest);
}

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.redaction.v1.server.client.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntityRecogintionEntity {
private String value;
private int startOffset;
private int endOffset;
private String type;
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java Normal file
View File

@ -0,0 +1,18 @@
package com.iqser.red.service.redaction.v1.server.client.model;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntityRecognitionRequest {
private List<EntityRecognitionSection> data;
}

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResponse.java Normal file
View File

@ -0,0 +1,21 @@
package com.iqser.red.service.redaction.v1.server.client.model;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntityRecognitionResponse {
@Builder.Default
private Map<String, List<EntityRecogintionEntity>> result = new HashMap<>();
}

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java Normal file
View File

@ -0,0 +1,20 @@
package com.iqser.red.service.redaction.v1.server.client.model;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntityRecognitionResult {
@Builder.Default
private Map<Integer, List<EntityRecogintionEntity>> entities = new HashMap<>();
}

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionSection.java Normal file
View File

@ -0,0 +1,16 @@
package com.iqser.red.service.redaction.v1.server.client.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EntityRecognitionSection {
private int sectionNumber;
private String text;
}

33
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -1,11 +1,7 @@
package com.iqser.red.service.redaction.v1.server.controller;
import com.iqser.red.service.file.management.v1.api.model.FileType;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
@ -13,8 +9,9 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@ -41,7 +38,7 @@ public class RedactionController implements RedactionResource {
private final AnnotationService annotationService;
private final PdfSegmentationService pdfSegmentationService;
private final RedactionStorageService redactionStorageService;
private final RedactionLogCreatorService redactionLogCreatorService;
private final RedactionLogMergeService redactionLogMergeService;
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
@ -143,14 +140,6 @@ public class RedactionController implements RedactionResource {
}
@Override
public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) {
droolsExecutionService.updateRules(dossierTemplateId);
}
@Override
public void testRules(@RequestBody String rules) {
@ -158,13 +147,21 @@ public class RedactionController implements RedactionResource {
}
@Override
public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) {
public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
log.info("Requested preview for: {}", redactionRequest);
dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions());
log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
if (redactionLog.getAnalysisVersion() == 0) {
// old redaction logs are returned directly
return redactionLog;
} else {
return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
}
}
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {

27
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java
View File

@ -189,18 +189,16 @@ public class PDFLinesTextStripper extends PDFTextStripper {
COSName objectName = (COSName) arguments.get(0);
PDXObject xobject = getResources().getXObject(objectName);
if (xobject instanceof PDImageXObject) {
PDImageXObject pdfImage = (PDImageXObject) xobject;
PDImageXObject image = (PDImageXObject)xobject;
Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix();
Rectangle2D imageBounds = calculateImagePosition(pdfImage);
Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
.getWidth(), (float) imageBounds.getHeight());
Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY());
// Memory Hack - sofReference kills me
FieldUtils.writeField(pdfImage, "cachedImageSubsampling", -1, true);
FieldUtils.writeField(image, "cachedImageSubsampling", -1, true);
if (rect.getHeight() > 2 && rect.getWidth() > 2) {
this.images.add(new PdfImage(pdfImage.getImage(), rect, pageNumber));
this.images.add(new PdfImage(image.getImage(), rect, pageNumber, image.getImage().getColorModel().hasAlpha()));
}
}
} catch (Exception e) {
@ -209,21 +207,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
}
private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException {
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds();
AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform());
imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
imageTransform.translate(0, -pdfImage.getHeight());
AffineTransform pageTransform = new AffineTransform();
pageTransform.concatenate(imageTransform);
return pageTransform.createTransformedShape(imageBounds).getBounds2D();
}
private float floatValue(COSBase value) {

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java
View File

@ -20,5 +20,6 @@ public class Image implements ReasonHolder {
private int sectionNumber;
private String section;
private int page;
private boolean hasTransparency;
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java
View File

@ -18,14 +18,16 @@ public class PdfImage {
private RedRectangle2D position;
private ImageType imageType;
private boolean isAppendedToParagraph;
private boolean hasTransparency;
@NonNull
private int page;
public PdfImage(BufferedImage image, Rectangle2D position, int page) {
public PdfImage(BufferedImage image, Rectangle2D position, int page, boolean hasTransparency) {
this.image = image;
this.position = new RedRectangle2D(position.getX(), position.getY(), position.getWidth(), position.getHeight());
this.page = page;
this.hasTransparency = hasTransparency;
}
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -72,6 +72,19 @@ public class Section {
}
public boolean fileAttributeByIdEqualsIgnoreCase(String id, String value){
return fileAttributes != null && fileAttributes.stream().filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
}
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(String placeholder, String value){
return fileAttributes != null && fileAttributes.stream().filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
}
public boolean fileAttributeByLabelEqualsIgnoreCase(String label, String value){
return fileAttributes != null && fileAttributes.stream().filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue())).findFirst().isPresent();
}
public boolean rowEquals(String headerName, String value) {
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");

42
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java
View File

@ -1,56 +1,30 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class AnalyzeResponseService {
private final RedactionServiceSettings redactionServiceSettings;
public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount,
RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) {
RedactionLog redactionLog, boolean hasUpdates) {
boolean hasHints = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> !entry.isExcluded())
.anyMatch(entry -> entry.isHint() && !entry.getType().equals("false_positive"));
boolean hasRequests = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> !entry.isExcluded())
.anyMatch(entry -> entry.isManual() && entry.getStatus()
.equals(com.iqser.red.service.redaction.v1.model.Status.REQUESTED));
boolean hasRedactions = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> !entry.isExcluded())
.anyMatch(entry -> entry.isRedacted() && !entry.isManual() || entry.isManual() && entry.getStatus()
.equals(com.iqser.red.service.redaction.v1.model.Status.APPROVED));
boolean hasImages = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> !entry.isExcluded())
.anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage());
boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog
.getRedactionLogEntry()
.isEmpty() && redactionChangeLog.getRedactionLogEntry()
.stream()
.anyMatch(entry -> !entry.getType().equals("false_positive"));
return AnalyzeResult.builder()
.dossierId(dossierId)
.fileId(fileId)
.duration(duration)
.numberOfPages(pageCount)
.hasHints(hasHints)
.hasRedactions(hasRedactions)
.hasRequests(hasRequests)
.hasImages(hasImages)
.hasUpdates(hasUpdates)
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.rulesVersion(redactionLog.getRulesVersion())
.dictionaryVersion(redactionLog.getDictionaryVersion())
.legalBasisVersion(redactionLog.getLegalBasisVersion())

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -33,6 +33,7 @@ public class DictionaryService {
public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
log.info("Updating dictionary data for: {} / {}", dossierTemplateId, dossierId);
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
@ -163,6 +164,7 @@ public class DictionaryService {
public float[] getColor(String type, String dossierTemplateId) {
log.info("requested : {} / {}",type,dossierTemplateId);
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) {
return model.getColor();
@ -210,13 +212,13 @@ public class DictionaryService {
dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
}
return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
return new Dictionary(copy.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
}
public float[] getRequestRemoveColor(String dossierTemplateId) {
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor();
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -63,7 +63,7 @@ public class DroolsExecutionService {
rulesVersion = -1L;
}
if (version > rulesVersion.longValue()) {
if (version > rulesVersion) {
rulesVersionPerDossierTemplateId.put(dossierTemplateId, version);
return createOrUpdateKieContainer(dossierTemplateId);
}
@ -128,7 +128,7 @@ public class DroolsExecutionService {
if (rulesVersion == null) {
return -1;
}
return rulesVersion.longValue();
return rulesVersion;
}
}

197
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -1,19 +1,23 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.*;
import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionResponse;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
@ -32,9 +36,12 @@ public class EntityRedactionService {
private final DictionaryService dictionaryService;
private final DroolsExecutionService droolsExecutionService;
private final SurroundingWordsService surroundingWordsService;
private final EntityRecognitionClient entityRecognitionClient;
private final RedactionServiceSettings redactionServiceSettings;
public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId, List<FileAttribute> fileAttributes) {
public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions,
String dossierId, List<FileAttribute> fileAttributes) {
dictionaryService.updateDictionary(dossierTemplateId, dossierId);
KieContainer container = droolsExecutionService.updateRules(dossierTemplateId);
@ -46,35 +53,13 @@ public class EntityRedactionService {
if (dictionary.hasLocalEntries()) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
documentEntities.stream().forEach(entity -> {
if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>())
.add(entity);
}
});
Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(documentEntities, dictionary);
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber, fileAttributes);
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
}
for (Entity entity : documentEntities) {
Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
.add(entityPositionSequence);
}
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
classifiedDoc.getEntities()
.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
}
}
classifiedDoc.setEntities(convertToEnititesPerPage(documentEntities));
dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId);
@ -83,9 +68,44 @@ public class EntityRedactionService {
}
public Map<Integer, List<Entity>> convertToEnititesPerPage(Set<Entity> entities) {
Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
for (Entity entity : entities) {
Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
.add(entityPositionSequence);
}
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
}
}
return entitiesPerPage;
}
public Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
entities.stream().forEach(entity -> {
if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
}
});
return hintsPerSectionNumber;
}
private Set<Entity> findEntities(Document classifiedDoc, KieContainer kieContainer,
ManualRedactions manualRedactions, Dictionary dictionary, boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber, List<FileAttribute> fileAttributes) {
Map<Integer, Set<Entity>> hintsPerSectionNumber,
List<FileAttribute> fileAttributes) {
Set<Entity> documentEntities = new HashSet<>();
@ -96,9 +116,9 @@ public class EntityRedactionService {
List<Table> tables = paragraph.getTables();
for (Table table : tables) {
if (table.getColCount() == 2) {
sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
} else {
sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
}
sectionNumber.incrementAndGet();
}
@ -125,22 +145,31 @@ public class EntityRedactionService {
}
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
documentEntities.addAll(analysedRowSection.getEntities());
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
documentEntities.addAll(analysedSection.getEntities());
for (Image image : analysedRowSection.getImages()) {
for (Image image : analysedSection.getImages()) {
classifiedDoc.getImages().computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
}
analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
addLocalValuesToDictionary(analysedSection, dictionary);
});
return documentEntities;
}
public void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {
analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> {
if (dictionary.isRecommendation(key)) {
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
if (!dictionary.containsValue(key, value)) {
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
}
});
} else {
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
if (dictionary.getLocalAccessMap().get(key) == null) {
log.warn("Dictionary {} is null", key);
@ -154,18 +183,14 @@ public class EntityRedactionService {
});
}
});
});
return documentEntities;
}
private List<SectionSearchableTextPair> processTablePerRow(Document classifiedDoc, Table table,
ManualRedactions manualRedactions,
AtomicInteger sectionNumber, Dictionary dictionary,
boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber, List<FileAttribute> fileAttributes) {
Map<Integer, Set<Entity>> hintsPerSectionNumber,
List<FileAttribute> fileAttributes) {
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
@ -190,7 +215,6 @@ public class EntityRedactionService {
sectionText.getSectionAreas().add(sectionArea);
sectionText.getTextBlocks().addAll(cell.getTextBlocks());
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
int cellStart = start;
if (!cell.isHeaderCell()) {
@ -251,7 +275,6 @@ public class EntityRedactionService {
private List<SectionSearchableTextPair> processTableAsOneText(Document classifiedDoc, Table table,
ManualRedactions manualRedactions,
AtomicInteger sectionNumber, Dictionary dictionary,
boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
@ -280,7 +303,6 @@ public class EntityRedactionService {
for (TextBlock textBlock : cell.getTextBlocks()) {
entireTableText.addAll(textBlock.getSequences());
}
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
}
}
@ -334,13 +356,12 @@ public class EntityRedactionService {
sectionText.setSectionNumber(sectionNumber.intValue());
sectionText.setTable(false);
sectionText.setImages(images.stream()
.map(image -> convert(image, sectionNumber.intValue(), headline))
.map(image -> convertAndRecategorize(image, sectionNumber.intValue(), headline, manualRedactions))
.collect(Collectors.toSet()));
sectionText.setTextBlocks(paragraphTextBlocks);
classifiedDoc.getSectionText().add(sectionText);
}
addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
@ -357,7 +378,7 @@ public class EntityRedactionService {
.searchableText(searchableText)
.dictionary(dictionary)
.images(images.stream()
.map(image -> convert(image, sectionNumber.intValue(), headline))
.map(image -> convertAndRecategorize(image, sectionNumber.intValue(), headline, manualRedactions))
.collect(Collectors.toSet()))
.fileAttributes(fileAttributes)
.build(), searchableText);
@ -376,39 +397,26 @@ public class EntityRedactionService {
String lowercaseInputString = searchableString.toLowerCase();
for (DictionaryModel model : dictionary.getDictionaryModels()) {
if (model.isCaseInsensitive()) {
found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model
.isDossierDictionary()));
} else {
found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model
.isDossierDictionary()));
}
}
if (redactionServiceSettings.isEnableEntityRecognition() && !local) {
found.addAll(getAiEntities(sectionNumber, searchableString, headline));
}
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
}
private void addSectionToManualRedactions(List<TextBlock> textBlocks, ManualRedactions manualRedactions,
String section, int sectionNumber) {
private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline,
ManualRedactions manualRedactions) {
if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) {
return;
}
for (TextBlock textBlock : textBlocks) {
for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
if (textBlock.contains(rectangle)) {
manualRedactionEntry.setSection(section);
manualRedactionEntry.setSectionNumber(sectionNumber);
}
}
}
}
}
private Image convert(PdfImage pdfImage, int sectionNumber, String headline) {
return Image.builder()
Image image = Image.builder()
.type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType()
.name()
.toLowerCase(Locale.ROOT))
@ -416,7 +424,44 @@ public class EntityRedactionService {
.sectionNumber(sectionNumber)
.section(headline)
.page(pdfImage.getPage())
.hasTransparency(pdfImage.isHasTransparency())
.build();
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
if (manualRedactions != null && manualRedactions.getImageRecategorizations() != null) {
for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorizations()) {
if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId()
.equals(imageId)) {
image.setType(imageRecategorization.getType());
}
}
}
return image;
}
private Set<Entity> getAiEntities(int sectionNumber, String searchableString, String headline) {
Set<Entity> found = new HashSet<>();
Map<String, Map<String, List<EntityRecogintionEntity>>> response = entityRecognitionClient.findAuthors(EntityRecognitionRequest
.builder()
.data(List.of(EntityRecognitionSection.builder()
.sectionNumber(sectionNumber)
.text(searchableString)
.build()))
.build());
EntityRecognitionResponse entityRecognitionResponse = new EntityRecognitionResponse(response.get("result:"));
if (entityRecognitionResponse.getResult() != null && entityRecognitionResponse.getResult()
.containsKey(String.valueOf(sectionNumber))) {
entityRecognitionResponse.getResult().get(String.valueOf(sectionNumber)).forEach(res -> {
found.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false));
});
}
return found;
}
}

253
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java
View File

@ -1,54 +1,32 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.file.management.v1.api.model.FileType;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Service
@RequiredArgsConstructor
@ -64,6 +42,7 @@ public class ReanalyzeService {
private final RedactionChangeLogService redactionChangeLogService;
private final AnalyzeResponseService analyzeResponseService;
private final LegalBasisClient legalBasisClient;
private final RedactionServiceSettings redactionServiceSettings;
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
@ -85,13 +64,12 @@ public class ReanalyzeService {
entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest
.getDossierId(), analyzeRequest.getFileAttributes());
redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest
.getDossierTemplateId());
redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getDossierTemplateId());
log.info("Redaction analysis successful...");
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion()
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion()
.getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion()
.getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
@ -100,9 +78,9 @@ public class ReanalyzeService {
log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc
.getDictionaryVersion(), analyzeRequest.getDossierTemplateId());
// first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
// store redactionLog
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
redactionLog = redactionLogChange.getRedactionLog();
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc
.getSectionText()));
@ -110,7 +88,7 @@ public class ReanalyzeService {
.getSectionGrid());
long duration = System.currentTimeMillis() - startTime;
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog);
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, redactionLogChange.isHasChanges());
}
@ -130,20 +108,66 @@ public class ReanalyzeService {
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions());
Map<String, List<Comment>> comments = null;
Set<ManualRedactionEntry> manualAdds = null;
Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() :
findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
if (analyzeRequest.getManualRedactions() != null) {
// TODO comments will be removed from redactionLog, so we ignore this first.
comments = analyzeRequest.getManualRedactions().getComments();
manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd();
if (sectionsToReanalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
}
List<SectionText> reanalysisSections = text.getSectionTexts()
.stream()
.filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
.collect(Collectors.toList());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
.getDossierId());
Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
Set<Entity> entities = findEntities(reanalysisSections, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage);
if (dictionary.hasLocalEntries()) {
Map<Integer, Set<Entity>> hintsPerSectionNumber = entityRedactionService.getHintsPerSection(entities, dictionary);
Set<Entity> foundByLocal = findEntities(reanalysisSections, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage);
EntitySearchUtils.addEntitiesWithHigherRank(entities, foundByLocal, dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
}
Map<Integer, List<Entity>> entitiesPerPage = entityRedactionService.convertToEnititesPerPage(entities);
List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
for (int page = 1; page <= text.getNumberOfPages(); page++) {
if (entitiesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest
.getDossierTemplateId()));
}
if (imagesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest
.getDossierTemplateId()));
}
}
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
analyzeResult.setWasReanalyzed(true);
return analyzeResult;
}
private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog,
Text text, AnalyzeRequest analyzeRequest) {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
Set<Integer> sectionsToReanalyse = new HashSet<>();
Map<Integer, Set<Image>> imageEntries = new HashMap<>();
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) {
if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
sectionsToReanalyse.add(entry.getSectionNumber());
}
if (entry.isImage() || entry.getType().equals("image")) {
@ -157,47 +181,24 @@ public class ReanalyzeService {
sectionsToReanalyse.add(sectionText.getSectionNumber());
}
if (manualAdds != null) {
for (SectionArea sectionArea : sectionText.getSectionAreas()) {
for (ManualRedactionEntry manualAdd : manualAdds) {
for (Rectangle manualPosition : manualAdd.getPositions()) {
if (sectionArea.contains(manualPosition)) {
manualAdd.setSection(sectionText.getHeadline());
manualAdd.setSectionNumber(sectionText.getSectionNumber());
}
}
}
}
}
}
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
return sectionsToReanalyse;
}
List<SectionText> reanalysisSections = new ArrayList<>();
for (SectionText sectionText : text.getSectionTexts()) {
if (sectionsToReanalyse.contains(sectionText.getSectionNumber())) {
reanalysisSections.add(sectionText);
}
}
//--
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
.getDossierId());
private Set<Entity> findEntities(List<SectionText> reanalysisSections, Dictionary dictionary,
KieContainer kieContainer, AnalyzeRequest analyzeRequest, boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
Map<Integer, Set<Image>> imagesPerPage) {
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
for (SectionText reanalysisSection : reanalysisSections) {
Set<Entity> entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection
.getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false);
.getHeadline(), reanalysisSection.getSectionNumber(), dictionary, local);
if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection
.getCellStarts());
@ -205,10 +206,28 @@ public class ReanalyzeService {
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
}
if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
.getImageRecategorizations() != null) {
for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
.getImageRecategorizations()) {
if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId()
.equals(imageId)) {
image.setType(imageRecategorization.getType());
}
}
}
}
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
.isLocal(false)
.dictionaryTypes(dictionary.getTypes())
.entities(entities)
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream
.concat(entities.stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber())
.stream())
.collect(Collectors.toSet()) : entities)
.text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
.searchText(reanalysisSection.getSearchableText().toString())
.headline(reanalysisSection.getHeadline())
@ -222,56 +241,19 @@ public class ReanalyzeService {
}
Set<Entity> entities = new HashSet<>();
Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
entities.addAll(analysedRowSection.getEntities());
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
entities.addAll(analysedSection.getEntities());
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
for (Image image : analysedRowSection.getImages()) {
for (Image image : analysedSection.getImages()) {
imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
}
entityRedactionService.addLocalValuesToDictionary(analysedSection, dictionary);
});
Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
for (Entity entity : entities) {
Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
.add(entityPositionSequence);
}
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
}
}
List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
for (int page = 1; page <= text.getNumberOfPages(); page++) {
if (entitiesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
.getDossierTemplateId()));
}
if (imagesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
.getDossierTemplateId()));
}
newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest
.getDossierTemplateId()));
}
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
AnalyzeResult analyzeResult = finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
analyzeResult.setWasReanalyzed(true);
return analyzeResult;
return entities;
}
@ -284,25 +266,29 @@ public class ReanalyzeService {
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
long duration = System.currentTimeMillis() - startTime;
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text
.getNumberOfPages(), redactionLog, changeLog);
.getNumberOfPages(), redactionLogChange.getRedactionLog(), redactionLogChange.isHasChanges());
}
private Set<String> getForceAndRemoveIds(ManualRedactions manualRedactions) {
private Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
if (manualRedactions == null) {
return new HashSet<>();
}
return Stream.concat(manualRedactions.getManualLegalBasisChanges().stream().map(ManualLegalBasisChange::getId),
Stream.concat(manualRedactions.getImageRecategorizations().stream().map(ManualImageRecategorization::getId),
Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))))
return Stream.concat(manualRedactions.getManualLegalBasisChanges()
.stream()
.map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations()
.stream()
.map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove()
.stream()
.map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))))
.collect(Collectors.toSet());
}
@ -318,21 +304,18 @@ public class ReanalyzeService {
.sectionNumber(entry.getSectionNumber())
.section(entry.getSection())
.page(position.getPage())
.hasTransparency(entry.isImageHasTransparency())
.build();
}
private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {
redactionLog.getRedactionLogEntry().forEach(entry -> {
entry.getPositions().forEach(pos -> {
if (excludedPages != null && excludedPages.contains(pos.getPage())) {
if(excludedPages != null && !excludedPages.isEmpty()) {
redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { if (excludedPages.contains(pos.getPage())) {
entry.setExcluded(true);
} else {
entry.setExcluded(false);
}}));
}
});
});
}
}

143
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java
View File

@ -1,19 +1,25 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.file.management.v1.api.model.FileType;
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import com.iqser.red.service.redaction.v1.model.Change;
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogChanges;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@ -22,76 +28,77 @@ public class RedactionChangeLogService {
private final RedactionStorageService redactionStorageService;
public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) {
try {
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) {
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog);
redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog);
return changeLog;
} catch (Exception e) {
log.debug("Previous redaction log not available");
return null;
}
}
private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) {
if (previousRedactionLog == null) {
return null;
currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
});
return new RedactionLogChanges(currentRedactionLog, false);
}
List<RedactionLogEntry> added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry());
added.removeAll(previousRedactionLog.getRedactionLogEntry());
List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> !entry.getChanges()
.get(entry.getChanges().size() - 1)
.getType()
.equals(ChangeType.REMOVED))
.collect(Collectors.toList());
List<RedactionLogEntry> removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry());
Set<RedactionLogEntry> added = new HashSet<>(currentRedactionLog.getRedactionLogEntry());
added.removeAll(notRemovedPreviousEntries);
Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
removed.removeAll(currentRedactionLog.getRedactionLogEntry());
List<RedactionChangeLogEntry> changeLogEntries = added.stream()
.map(entry -> convert(entry, ChangeType.ADDED))
.collect(Collectors.toList());
changeLogEntries.addAll(removed.stream()
.map(entry -> convert(entry, ChangeType.REMOVED))
.collect(Collectors.toList()));
Map<String, RedactionLogEntry> addedIds = new HashMap<>();
added.forEach(entry -> {
addedIds.put(entry.getId(), entry);
});
return new RedactionChangeLog(changeLogEntries,
currentRedactionLog.getDictionaryVersion(),
currentRedactionLog.getDossierDictionaryVersion(),
currentRedactionLog.getRulesVersion(),
currentRedactionLog.getLegalBasisVersion());
Set<String> removedIds = new HashSet<>();
removed.forEach(entry -> {
removedIds.add(entry.getId());
});
List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
List<RedactionLogEntry> toRemove = new ArrayList<>();
newRedactionLogEntries.forEach(entry -> {
if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges();
changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now()));
var newEntry = addedIds.get(entry.getId());
newEntry.setChanges(changes);
addedIds.put(entry.getId(), newEntry);
toRemove.add(entry);
} else if (removedIds.contains(entry.getId())) {
entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now()));
} else if (addedIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges();
changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
var newEntry = addedIds.get(entry.getId());
newEntry.setChanges(changes);
addedIds.put(entry.getId(), newEntry);
toRemove.add(entry);
}
});
newRedactionLogEntries.removeAll(toRemove);
private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) {
addedIds.forEach((k, v) -> {
if(v.getChanges().isEmpty()) {
v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
}
newRedactionLogEntries.add(v);
});
return RedactionChangeLogEntry.builder()
.id(entry.getId())
.type(entry.getType())
.value(entry.getValue())
.reason(entry.getReason())
.matchedRule(entry.getMatchedRule())
.legalBasis(entry.getLegalBasis())
.redacted(entry.isRedacted())
.isHint(entry.isHint())
.isRecommendation(entry.isRecommendation())
.section(entry.getSection())
.color(entry.getColor())
.positions(entry.getPositions())
.sectionNumber(entry.getSectionNumber())
.manual(entry.isManual())
.status(entry.getStatus())
.manualRedactionType(entry.getManualRedactionType())
.isDictionaryEntry(entry.isDictionaryEntry())
.textBefore(entry.getTextBefore())
.textAfter(entry.getTextAfter())
.comments(entry.getComments())
.changeType(changeType)
.isDossierDictionaryEntry(entry.isDossierDictionaryEntry())
.excluded(entry.isExcluded())
.build();
currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty());
}
}

457
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -1,6 +1,10 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.model.CellRectangle;
import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@ -9,17 +13,19 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.ReasonHolder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@ -29,10 +35,7 @@ public class RedactionLogCreatorService {
private final DictionaryService dictionaryService;
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
String dossierTemplateId) {
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
public void createRedactionLog(Document classifiedDoc, int numberOfPages, String dossierTemplateId) {
for (int page = 1; page <= numberOfPages; page++) {
@ -40,24 +43,18 @@ public class RedactionLogCreatorService {
if (classifiedDoc.getEntities().get(page) != null) {
classifiedDoc.getRedactionLogEntities()
.addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId));
}
if (manualRedactionPages.contains(page)) {
classifiedDoc.getRedactionLogEntities()
.addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId));
.addAll(addEntries(classifiedDoc.getEntities(), page, dossierTemplateId));
}
if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) {
classifiedDoc.getRedactionLogEntities()
.addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId));
.addAll(addImageEntries(classifiedDoc.getImages(), page, dossierTemplateId));
}
}
}
public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions,
int pageNumber, String dossierTemplateId) {
public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, int pageNumber, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -67,7 +64,7 @@ public class RedactionLogCreatorService {
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
.id(id)
.color(getColorForImage(image.getType(), dossierTemplateId, false, image.isRedaction()))
.color(getColor(image.getType(), dossierTemplateId, image.isRedaction()))
.isImage(true)
.type(image.getType())
.redacted(image.isRedaction())
@ -83,9 +80,9 @@ public class RedactionLogCreatorService {
.getWidth(), (float) image.getPosition().getHeight(), pageNumber)))
.sectionNumber(image.getSectionNumber())
.section(image.getSection())
.imageHasTransparency(image.isHasTransparency())
.build();
processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry);
redactionLogEntities.add(redactionLogEntry);
}
@ -93,138 +90,8 @@ public class RedactionLogCreatorService {
return redactionLogEntities;
}
private void processImageEntry(ManualRedactions manualRedactions, String dossierTemplateId, ReasonHolder image, RedactionLogEntry redactionLogEntry) {
if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) {
for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) {
if (recategorization.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (recategorization.getStatus().equals(Status.APPROVED)) {
image.setRedaction(recategorization.isRedacted());
redactionLogEntry.setType(recategorization.getType());
redactionLogEntry.setHint(dictionaryService.isHint(recategorization.getType(), dossierTemplateId));
redactionLogEntry.setRedacted(recategorization.isRedacted());
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setLegalBasis(recategorization.getLegalBasis());
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override");
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
} else if (recategorization.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setRecategorizationType(recategorization.getType());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
}
}
}
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
if (manualRemoval.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
image.setRedaction(false);
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", removed by manual override");
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
}
}
}
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
if (manualForceRedact.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
image.setRedaction(true);
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
}
}
if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) {
for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) {
if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
}
}
}
private Set<Integer> getManualRedactionPages(ManualRedactions manualRedactions) {
Set<Integer> manualRedactionPages = new HashSet<>();
if (manualRedactions == null) {
return manualRedactionPages;
}
manualRedactions.getEntriesToAdd().forEach(entry -> {
entry.getPositions().forEach(pos -> {
manualRedactionPages.add(pos.getPage());
});
});
return manualRedactionPages;
}
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions,
int page, String dossierTemplateId) {
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -247,7 +114,6 @@ public class RedactionLogCreatorService {
}
redactionLogEntry.setId(entityPositionSequence.getId());
processRedactionLogEntry(manualRedactions, dossierTemplateId, redactionLogEntry, entity);
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
@ -271,109 +137,6 @@ public class RedactionLogCreatorService {
return redactionLogEntities;
}
private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry, ReasonHolder reasonHolder) {
List<Comment> comments = null;
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
if (manualRemoval.getId().equals(redactionLogEntry.getId())) {
comments = manualRedactions.getComments().get(manualRemoval.getId());
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
reasonHolder.setRedaction(false);
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
}
}
}
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
if (manualForceRedact.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
reasonHolder.setRedaction(true);
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
}
}
if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) {
for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) {
if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
}
}
if (manualRedactions != null) {
comments = manualRedactions.getComments().get(redactionLogEntry.getId());
}
redactionLogEntry.setComments(comments);
}
private String mergeReasonIfNecessary(String currentReason, String addition) {
if (currentReason != null) {
if (!currentReason.contains(addition)) {
return currentReason + addition;
}
return currentReason;
} else {
return "";
}
}
private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
@ -402,73 +165,10 @@ public class RedactionLogCreatorService {
}
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments, int page,
String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
if (manualAdds == null) {
return redactionLogEntities;
}
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
String id = manualRedactionEntry.getId();
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId);
List<Rectangle> rectanglesOnPage = new ArrayList<>();
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
if (page == rectangle.getPage()) {
rectanglesOnPage.add(rectangle);
redactionLogEntry.getPositions().add(rectangle);
}
}
redactionLogEntry.setComments(comments.get(id));
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
redactionLogEntities.add(redactionLogEntry);
}
}
return redactionLogEntities;
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary();
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String dossierTemplateId) {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType())
.redacted(true)
.isHint(false)
.section(manualRedactionEntry.getSection())
.sectionNumber(manualRedactionEntry.getSectionNumber())
.manual(true)
.status(manualRedactionEntry.getStatus())
.manualRedactionType(ManualRedactionType.ADD)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.build();
}
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, false, entity.isRedaction()))
.color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
.reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis())
.value(entity.getWord())
@ -489,59 +189,6 @@ public class RedactionLogCreatorService {
}
private float[] getColor(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) {
if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
}
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
if (status.equals(Status.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return getColor(type, dossierTemplateId);
}
private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForImage(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) {
if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
}
if (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private boolean isHint(String type, String dossierTemplateId) {
return dictionaryService.isHint(type, dossierTemplateId);
}
private boolean isRecommendation(String type, String dossierTemplateId) {
return dictionaryService.isRecommendation(type, dossierTemplateId);
}
private void addSectionGrid(Document classifiedDoc, int page) {
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
@ -585,74 +232,26 @@ public class RedactionLogCreatorService {
}
}
public RedactionLog getRedactionLogPreview(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) {
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
var manualRedactionPages = getManualRedactionPages(manualRedactions);
// generate all manual entries
var manualRedactionLogEntries = new HashMap<String, RedactionLogEntry>();
for (var page : manualRedactionPages) {
var pageEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId);
for (var entry : pageEntries) {
manualRedactionLogEntries.put(entry.getId(), entry);
}
}
for (var manualEntry : manualRedactionLogEntries.values()) {
var existingEntry = redactionLog.getRedactionLogEntry().stream().filter(e -> e.getId().equals(manualEntry.getId())).findAny();
if (existingEntry.isPresent()) {
// if it has already been processed of sorts, update it
BeanUtils.copyProperties(manualEntry, existingEntry.get());
} else {
// not yet in the redaction-log - add it
redactionLog.getRedactionLogEntry().add(manualEntry);
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
private boolean isHint(String type, String dossierTemplateId) {
var reasonHolder = new PreviewReasonHolder(entry);
if (entry.isImage()) {
processImageEntry(manualRedactions, dossierTemplateId, reasonHolder, entry);
return dictionaryService.isHint(type, dossierTemplateId);
}
processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder);
private boolean isRecommendation(String type, String dossierTemplateId) {
return dictionaryService.isRecommendation(type, dossierTemplateId);
}
return redactionLog;
}
public static class PreviewReasonHolder implements ReasonHolder {
private final RedactionLogEntry entry;
public PreviewReasonHolder(RedactionLogEntry entry) {
this.entry = entry;
}
@Override
public String getRedactionReason() {
return entry.getReason();
}
@Override
public void setRedactionReason(String reason) {
entry.setReason(reason);
}
@Override
public boolean isRedaction() {
return entry.isRedacted();
}
@Override
public void setRedaction(boolean value) {
entry.setRedacted(value);
}
}
}

328
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java Normal file
View File

@ -0,0 +1,328 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.Status;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class RedactionLogMergeService {
private final DictionaryService dictionaryService;
public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId,
ManualRedactions manualRedactions, Set<Integer> excludedPages) {
log.info("Merging Redaction log with manual redactions {}", manualRedactions);
if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
processRedactionLogEntry(manualRedactionWrappers.stream()
.filter(mr -> entry.getId().equals(mr.getId()))
.collect(Collectors.toList()), dossierTemplateId, entry);
entry.setComments(manualRedactions.getComments().get(entry.getId()));
if (excludedPages != null && !excludedPages.isEmpty()) {
entry.getPositions().forEach(pos -> {
if (excludedPages.contains(pos.getPage())) {
entry.setExcluded(true);
}
});
}
}
}
return redactionLog;
}
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
manualRedactions.getImageRecategorizations().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
}
});
manualRedactions.getIdsToRemove().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
}
});
manualRedactions.getForceRedacts().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
}
});
manualRedactions.getManualLegalBasisChanges().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
}
});
Collections.sort(manualRedactionWrappers);
return manualRedactionWrappers;
}
private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers,
String dossierTemplateId, RedactionLogEntry redactionLogEntry) {
manualRedactionWrappers.forEach(mrw -> {
if (mrw.getItem() instanceof ManualImageRecategorization) {
var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setType(imageRecategorization.getType());
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
}
if (mrw.getItem() instanceof IdRemoval) {
var manualRemoval = (IdRemoval) mrw.getItem();
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), true));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser());
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
}
if (mrw.getItem() instanceof ManualForceRedact) {
var manualForceRedact = (ManualForceRedact) mrw.getItem();
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
if (mrw.getItem() instanceof ManualLegalBasisChange) {
var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
});
}
private String mergeReasonIfNecessary(String currentReason, String addition) {
if (currentReason != null) {
if (!currentReason.contains(addition)) {
return currentReason + addition;
}
return currentReason;
} else {
return "";
}
}
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry
.getId(), dossierTemplateId);
redactionLogEntry.setPositions(manualRedactionEntry.getPositions());
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId()));
redactionLogEntries.add(redactionLogEntry);
}
}
return redactionLogEntries;
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus()
.equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String dossierTemplateId) {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType())
.redacted(true)
.isHint(false)
.section(null)
.sectionNumber(-1)
.manual(true)
.status(manualRedactionEntry.getStatus())
.manualRedactionType(ManualRedactionType.ADD)
.manualRedactionUserId(manualRedactionEntry.getUser())
.build();
}
private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction,
boolean skipped) {
if (requested) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
}
if (skipped || !isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
if (status.equals(Status.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return getColor(type, dossierTemplateId);
}
private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, dossierTemplateId);
}
@Data
@AllArgsConstructor
private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
private String id;
private OffsetDateTime date;
private Object item;
@Override
public int compareTo(ManualRedactionWrapper o) {
return this.date.compareTo(o.date);
}
}
}

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -5,6 +5,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncre
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@ -17,7 +18,6 @@ import java.util.stream.Collectors;
@SuppressWarnings("PMD")
public class EntitySearchUtils {
public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {
String inputString = sectionText.toLowerCase(Locale.ROOT);
@ -38,10 +38,8 @@ public class EntitySearchUtils {
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
if (value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())) {
return true;
}
}
} while (startIndex > -1);
}
return false;
@ -98,8 +96,8 @@ public class EntitySearchUtils {
.sorted(Comparator.comparing(Entity::getStart))
.collect(Collectors.toList());
Entity firstEntity = orderedEntities.get(0);
List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity
.getType()), firstEntity.getTargetSequences());
List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
.trim(), dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
for (int i = 0; i <= orderedEntities.size() - 1; i++) {
try {
@ -133,6 +131,7 @@ public class EntitySearchUtils {
public void addEntitiesWithHigherRank(Set<Entity> entities, Set<Entity> found, Dictionary dictionary) {
found.forEach(f -> addEntitiesWithHigherRank(entities, f, dictionary));
}
@ -148,9 +147,11 @@ public class EntitySearchUtils {
entities.add(found);
}
public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found) {
// HashSet keeps old value but we want the new.
entities.removeAll(found);
entities.addAll(found);
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java
View File

@ -12,7 +12,7 @@ public class TextNormalizationUtilities {
* @return Text without line-break hyphenation.
*/
public static String removeHyphenLineBreaks(String text) {
return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R", "$1");
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageMergeService.java
View File

@ -118,7 +118,7 @@ public class ImageMergeService {
//set position for merged image with values of image1 and the height of both
Rectangle2D pos = new Rectangle2D.Float();
pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), rotation == 90 ? width + width2: width, rotation == 90 ? height1 : height1 + height2);
PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage());
PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage(), image1.isHasTransparency() || image2.isHasTransparency());
// Graphics need to be disposed
image1.getImage().flush();

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java
View File

@ -15,4 +15,8 @@ public class RedactionServiceSettings {
private float maxImageCropboxRatio = 0.9f;
private int analysisVersion = 1;
private boolean enableEntityRecognition = true;
}

1
redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml
View File

@ -4,6 +4,7 @@ info:
configuration-service.url: "http://configuration-service-v1:8080"
file-management-service.url: "http://file-management-service-v1:8080"
image-service.url: "http://image-service-v1:8080"
entity-recognition-service.url: "http://entity-recognition-service-v1:8080"
server:
port: 8080

119
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -18,7 +18,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.junit.After;
import org.junit.Before;
@ -84,7 +86,6 @@ public class RedactionIntegrationTest {
private static final String PII = "PII";
@Autowired
private RedactionController redactionController;
@ -127,6 +128,7 @@ public class RedactionIntegrationTest {
private final Map<String, Integer> rankTypeMap = new HashMap<>();
private final Colors colors = new Colors();
private final Map<String, Long> reanlysisVersions = new HashMap<>();
private final Set<String> deleted = new HashSet<>();
private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
private final static String TEST_DOSSIER_ID = "123";
@ -152,18 +154,20 @@ public class RedactionIntegrationTest {
return kieServices.newKieContainer(kieModule.getReleaseId());
}
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
}
}
@After
public void cleanupStorage() {
if (this.storageService instanceof FileSystemBackedStorageService) {
((FileSystemBackedStorageService) this.storageService).clearStorage();
}
@ -180,7 +184,8 @@ public class RedactionIntegrationTest {
loadDictionaryForTest();
loadTypeForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(0L);
when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse.builder()
when(dictionaryClient.getAllTypes(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(TypeResponse
.builder()
.types(getTypeResponse())
.build());
@ -197,26 +202,42 @@ public class RedactionIntegrationTest {
.build()))
.build());
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(PII, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PII, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(IMAGE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(IMAGE, false));
when(dictionaryClient.getDictionaryForType(OCR, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(OCR, false));
when(dictionaryClient.getDictionaryForType(LOGO, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(LOGO, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS, TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@ -478,7 +499,8 @@ public class RedactionIntegrationTest {
return DictionaryResponse.builder()
.hexColor(typeColorMap.get(type))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary
.get(type)))
.isHint(hintTypeMap.get(type))
.isCaseInsensitive(caseInSensitiveMap.get(type))
.isRecommendation(recommendationTypeMap.get(type))
@ -491,7 +513,8 @@ public class RedactionIntegrationTest {
List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
entries.forEach(entry -> {
dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, false));
dictionaryEntries.add(new DictionaryEntry(entry, reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L, deleted
.contains(entry) ? true : false));
});
return dictionaryEntries;
}
@ -499,6 +522,7 @@ public class RedactionIntegrationTest {
@Test
public void test270Rotated() {
AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf");
MemoryStats.printMemoryStats();
AnalyzeResult result = reanalyzeService.analyze(request);
@ -509,12 +533,14 @@ public class RedactionIntegrationTest {
@Test
@Ignore
public void testLargeScannedFileOOM() {
AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
MemoryStats.printMemoryStats();
AnalyzeResult result = reanalyzeService.analyze(request);
assertThat(result).isNotNull();
}
@Test
public void testMergedImages() throws IOException {
@ -553,14 +579,13 @@ public class RedactionIntegrationTest {
long rend = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (rend - rstart));
long end = System.currentTimeMillis();
System.out.println("duration: " + (end - start));
}
@Test
@Ignore
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
@ -636,7 +661,12 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
request.setExcludedPages(Set.of(1));
request.setFileAttributes(List.of(FileAttribute.builder().id("fileAttributeId").label("Vertebrate Study").placeholder("{fileattributes.vertebrateStudy}").value("true").build()));
request.setFileAttributes(List.of(FileAttribute.builder()
.id("fileAttributeId")
.label("Vertebrate Study")
.placeholder("{fileattributes.vertebrateStudy}")
.value("true")
.build()));
AnalyzeResult result = reanalyzeService.analyze(request);
@ -684,12 +714,21 @@ public class RedactionIntegrationTest {
dictionary.get(AUTHOR).add("physical");
reanlysisVersions.put("physical", 2L);
// dictionary.get(VERTEBRATE).add("s-metolachlor");
// reanlysisVersions.put("s-metolachlor", 3L);
deleted.add("David Chubb");
deleted.add("mouse");
dictionary.get(FALSE_POSITIVE).add("David Chubb");
reanlysisVersions.put("David Chubb", 3L);
reanlysisVersions.put("mouse", 3L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
start = System.currentTimeMillis();
@ -699,16 +738,12 @@ public class RedactionIntegrationTest {
.id("37eee3e9d589a5cc529bfec38c3ba479")
.status(Status.APPROVED)
.type("signature")
.redacted(true)
.legalBasis("Article 39(e)(1) and Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()));
request.setManualRedactions(manualRedactions);
AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request);
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
end = System.currentTimeMillis();
@ -723,6 +758,20 @@ public class RedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument());
}
deleted.remove("mouse");
reanlysisVersions.put("mouse", 4L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
reanalyzeService.reanalyze(request);
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
System.out.println("hi");
}
@ -778,7 +827,6 @@ public class RedactionIntegrationTest {
.status(Status.APPROVED)
.build()));
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
manualRedactions.getComments().put(manualAddId, List.of(comment));
@ -793,12 +841,10 @@ public class RedactionIntegrationTest {
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
request.setManualRedactions(manualRedactions);
AnalyzeResult result = reanalyzeService.analyze(request);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id("5b940b2cb401ed9f5be6fc24f6e77bcf")
@ -819,7 +865,6 @@ public class RedactionIntegrationTest {
.fileId(TEST_FILE_ID)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
}
@ -836,7 +881,6 @@ public class RedactionIntegrationTest {
System.out.println("classificationTest");
ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
RedactionRequest redactionRequest = RedactionRequest.builder()
@ -937,8 +981,10 @@ public class RedactionIntegrationTest {
});
}
@SneakyThrows
private AnalyzeRequest prepareStorage(String file) {
ClassPathResource pdfFileResource = new ClassPathResource(file);
return prepareStorage(pdfFileResource.getInputStream());
@ -970,7 +1016,6 @@ public class RedactionIntegrationTest {
long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf");
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
AnalyzeResult result = reanalyzeService.analyze(request);

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml
View File

@ -1,6 +1,7 @@
configuration-service.url: "http://configuration-service-v1:8080"
image-service.url: "http://image-service-v1:8080"
file-management-service.url: "http://file-management-service-v1:8080"
entity-recognition-service.url: "http://entity-recognition-service-v1:8080"
ribbon:
ConnectTimeout: 600000
@ -17,3 +18,4 @@ platform.multi-tenancy:
redaction-service:
enable-image-classification: false
enable-entity-recognition: false

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt
View File

@ -1676,7 +1676,6 @@ da Silva Rejane
Das R
Das, R.
Daughtry, CST
David Chubb
David Chubb|Lorraine Britton
David Clarke
Davies

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt
View File

@ -236,3 +236,4 @@ No details reported
Not available
Test facility
TBD
David Chubb