Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-2228: Use manual redaction classes from persistence-service, changed type in redactionlog to typeId

Browse Source
This commit is contained in:
Dominique Eifländer 2021-09-23 12:15:00 +02:00
parent d9b78643fb
commit 6ca134bdbd
18 changed files with 202 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -11,6 +11,8 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
@Data @Data
@Builder @Builder
@NoArgsConstructor @NoArgsConstructor

21
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Comment.java
View File

@ -1,21 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Comment {
private String id;
private OffsetDateTime date;
private String text;
private String user;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class IdRemoval {
private String id;
private String user;
private Status status;
private boolean removeFromDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualForceRedact {
private String id;
private String user;
private Status status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualImageRecategorization {
private String id;
private String user;
private Status status;
private String type;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualLegalBasisChange {
private String id;
private String user;
private Status status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

34
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java
View File

@ -1,34 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualRedactionEntry {
private String id;
private String user;
private String type;
private String value;
private String reason;
private String legalBasis;
private List<Rectangle> positions = new ArrayList<>();
private Status status;
private boolean addToDictionary;
private boolean addToDossierDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

38
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java
View File

@ -1,38 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualRedactions {
@Builder.Default
private Set<IdRemoval> idsToRemove = new HashSet<>();
@Builder.Default
private Set<ManualForceRedact> forceRedacts = new HashSet<>();
@Builder.Default
private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
@Builder.Default
private Set<ManualImageRecategorization> imageRecategorizations = new HashSet<>();
@Builder.Default
private Set<ManualLegalBasisChange> manualLegalBasisChanges = new HashSet<>();
@Builder.Default
private Map<String, List<Comment>> comments = new HashMap<>();
}

8
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -11,6 +11,8 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
@Data @Data
@Builder @Builder
@NoArgsConstructor @NoArgsConstructor
@ -19,7 +21,7 @@ import java.util.Set;
public class RedactionLogEntry { public class RedactionLogEntry {
private String id; private String id;
private String type; private String typeId;
private String value; private String value;
private String reason; private String reason;
private int matchedRule; private int matchedRule;
@ -34,7 +36,7 @@ public class RedactionLogEntry {
private List<Rectangle> positions = new ArrayList<>(); private List<Rectangle> positions = new ArrayList<>();
private int sectionNumber; private int sectionNumber;
private boolean manual; private boolean manual;
private Status status; private AnnotationStatus status;
private ManualRedactionType manualRedactionType; private ManualRedactionType manualRedactionType;
private String manualRedactionUserId; private String manualRedactionUserId;
private boolean isDictionaryEntry; private boolean isDictionaryEntry;
@ -43,7 +45,7 @@ public class RedactionLogEntry {
private String textAfter; private String textAfter;
@Builder.Default @Builder.Default
private List<Comment> comments = new ArrayList<>(); private List<com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment> comments = new ArrayList<>();
private int startOffset; private int startOffset;
private int endOffset; private int endOffset;

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.model;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

5
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Status.java
View File

@ -1,5 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
public enum Status {
REQUESTED, APPROVED, DECLINED
}

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -13,13 +13,14 @@ import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.persistence.service.v1.api.model.FileType; import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationId;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
@ -113,7 +114,7 @@ public class AnalyzeService {
dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId()); dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId());
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
.getDossierTemplateId()); .getDossierTemplateId(), analyzeRequest.getDossierId());
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary
@ -166,7 +167,7 @@ public class AnalyzeService {
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities); PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
.getDossierTemplateId()); .getDossierTemplateId(), analyzeRequest.getDossierId());
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
@ -186,7 +187,7 @@ public class AnalyzeService {
if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
sectionsToReanalyse.add(entry.getSectionNumber()); sectionsToReanalyse.add(entry.getSectionNumber());
} }
if (entry.isImage() || entry.getType().equals("image")) { if (entry.isImage()) {
imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry)); imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
} }
} }
@ -242,13 +243,13 @@ public class AnalyzeService {
return new HashSet<>(); return new HashSet<>();
} }
return Stream.concat(manualRedactions.getManualLegalBasisChanges() return Stream.concat(manualRedactions.getLegalBasisChanges()
.stream() .stream()
.map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations() .map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorization()
.stream() .stream()
.map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove() .map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove()
.stream() .stream()
.map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) .map(IdRemoval::getId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getId)))).map(AnnotationId::getId)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} }
@ -258,7 +259,7 @@ public class AnalyzeService {
Rectangle position = entry.getPositions().get(0); Rectangle position = entry.getPositions().get(0);
return Image.builder() return Image.builder()
.type(entry.getType()) .type(entry.getTypeId().split(":")[0])
.position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft() .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft()
.getY(), position.getWidth(), position.getHeight())) .getY(), position.getWidth(), position.getHeight()))
.sectionNumber(entry.getSectionNumber()) .sectionNumber(entry.getSectionNumber())

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java
View File

@ -12,6 +12,7 @@ import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.awt.Color; import java.awt.Color;
@ -91,7 +92,7 @@ public class AnnotationService {
if (redactionLogEntry.getComments() != null) { if (redactionLogEntry.getComments() != null) {
for (Comment comment : redactionLogEntry.getComments()) { for (Comment comment : redactionLogEntry.getComments()) {
PDAnnotationText txtAnnot = new PDAnnotationText(); PDAnnotationText txtAnnot = new PDAnnotationText();
txtAnnot.setAnnotationName(comment.getId()); txtAnnot.setAnnotationName(String.valueOf(comment.getId()));
txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
txtAnnot.setName(PDAnnotationText.NAME_COMMENT); txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime())); txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -14,10 +14,10 @@ import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.Status;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
@ -84,12 +84,12 @@ public class EntityRedactionService {
if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages() if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions() .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
.getImageRecategorizations() != null) { .getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) { for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
.getImageRecategorizations()) { .getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getId()
.equals(imageId)) { .equals(imageId)) {
image.setType(imageRecategorization.getType()); image.setType(imageRecategorization.getType());
} }

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -31,13 +31,13 @@ public class RedactionLogCreatorService {
public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages, public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages,
String dossierTemplateId) { String dossierTemplateId, String dossierId) {
List<RedactionLogEntry> entries = new ArrayList<>(); List<RedactionLogEntry> entries = new ArrayList<>();
for (int page = 1; page <= numberOfPages; page++) { for (int page = 1; page <= numberOfPages; page++) {
if (pageEntities.getEntitiesPerPage().get(page) != null) { if (pageEntities.getEntitiesPerPage().get(page) != null) {
entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId)); entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId, dossierId));
} }
if (pageEntities.getImagesPerPage().get(page) != null) { if (pageEntities.getImagesPerPage().get(page) != null) {
@ -62,7 +62,7 @@ public class RedactionLogCreatorService {
.id(id) .id(id)
.color(getColor(image.getType(), dossierTemplateId, image.isRedaction())) .color(getColor(image.getType(), dossierTemplateId, image.isRedaction()))
.isImage(true) .isImage(true)
.type(image.getType()) .typeId(IdBuilder.getTypeId(image.getType(), dossierTemplateId, null))
.redacted(image.isRedaction()) .redacted(image.isRedaction())
.reason(image.getRedactionReason()) .reason(image.getRedactionReason())
.legalBasis(image.getLegalBasis()) .legalBasis(image.getLegalBasis())
@ -86,7 +86,7 @@ public class RedactionLogCreatorService {
} }
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId) { public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId, String dossierId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>(); List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -98,7 +98,7 @@ public class RedactionLogCreatorService {
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId); RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId, dossierId);
if (processedIds.contains(entityPositionSequence.getId())) { if (processedIds.contains(entityPositionSequence.getId())) {
// TODO refactor this outer loop jump as soon as we have the time. // TODO refactor this outer loop jump as soon as we have the time.
@ -157,7 +157,7 @@ public class RedactionLogCreatorService {
} }
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId, String dossierId) {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId()))); entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId())));
@ -167,7 +167,7 @@ public class RedactionLogCreatorService {
.reason(entity.getRedactionReason()) .reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis()) .legalBasis(entity.getLegalBasis())
.value(entity.getWord()) .value(entity.getWord())
.type(entity.getType()) .typeId(IdBuilder.getTypeId(entity.getType(), dossierTemplateId, entity.isDossierDictionaryEntry() ? dossierId : null))
.redacted(entity.isRedaction()) .redacted(entity.isRedaction())
.isHint(isHint(entity.getType(), dossierTemplateId)) .isHint(isHint(entity.getType(), dossierTemplateId))
.isRecommendation(isRecommendation(entity.getType(), dossierTemplateId)) .isRecommendation(isRecommendation(entity.getType(), dossierTemplateId))
@ -206,4 +206,6 @@ public class RedactionLogCreatorService {
return dictionaryService.isRecommendation(type, dossierTemplateId); return dictionaryService.isRecommendation(type, dossierTemplateId);
} }
} }

119
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -10,17 +10,20 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Comment; import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact; import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.Status; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
@ -74,27 +77,27 @@ public class RedactionLogMergeService {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>(); List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
manualRedactions.getImageRecategorizations().forEach(item -> { manualRedactions.getImageRecategorization().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
} }
}); });
manualRedactions.getIdsToRemove().forEach(item -> { manualRedactions.getIdsToRemove().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
} }
}); });
manualRedactions.getForceRedacts().forEach(item -> { manualRedactions.getForceRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
} }
}); });
manualRedactions.getManualLegalBasisChanges().forEach(item -> { manualRedactions.getLegalBasisChanges().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
} }
}); });
@ -112,18 +115,18 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof ManualImageRecategorization) { if (mrw.getItem() instanceof ManualImageRecategorization) {
var imageRecategorization = (ManualImageRecategorization) mrw.getItem(); var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(Status.APPROVED)) { if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setType(imageRecategorization.getType()); redactionLogEntry.setTypeId(IdBuilder.getTypeId(imageRecategorization.getType(), dossierTemplateId, null));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) { } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser()); redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
@ -135,19 +138,19 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof IdRemoval) { if (mrw.getItem() instanceof IdRemoval) {
var manualRemoval = (IdRemoval) mrw.getItem(); var manualRemoval = (IdRemoval) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) { if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(false); redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry
.isRedacted(), true)); .isRedacted(), true));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setReason(manualOverrideReason);
@ -158,24 +161,24 @@ public class RedactionLogMergeService {
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
} }
if (mrw.getItem() instanceof ManualForceRedact) { if (mrw.getItem() instanceof ManualForceRedaction) {
var manualForceRedact = (ManualForceRedact) mrw.getItem(); var manualForceRedact = (ManualForceRedaction) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) { if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(true); redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser()); redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
@ -188,18 +191,18 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof ManualLegalBasisChange) { if (mrw.getItem() instanceof ManualLegalBasisChange) {
var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem(); var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED); redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false)); .isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else { } else {
redactionLogEntry.setStatus(Status.DECLINED); redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
@ -235,8 +238,8 @@ public class RedactionLogMergeService {
if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry
.getId(), dossierTemplateId); .getId().getId(), dossierTemplateId);
redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId()));
redactionLogEntries.add(redactionLogEntry); redactionLogEntries.add(redactionLogEntry);
} }
@ -245,11 +248,16 @@ public class RedactionLogMergeService {
return redactionLogEntries; return redactionLogEntries;
} }
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Rectangle> positions){
return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
.getHeight(), pos.getPage())).collect(Collectors.toList());
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus() return manualRedactionEntry.getStatus()
.equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); .equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
} }
@ -258,13 +266,13 @@ public class RedactionLogMergeService {
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(id) .id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) .color(getColorForManualAdd(getTypeFromTypeId(manualRedactionEntry.getTypeId()), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason()) .reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary()) .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis()) .legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue()) .value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType()) .typeId(manualRedactionEntry.getTypeId())
.redacted(true) .redacted(true)
.isHint(false) .isHint(false)
.section(null) .section(null)
@ -290,11 +298,11 @@ public class RedactionLogMergeService {
} }
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
if (status.equals(Status.REQUESTED)) { if (status.equals(AnnotationStatus.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId); return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) { } else if (status.equals(AnnotationStatus.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId); return dictionaryService.getNotRedactedColor(dossierTemplateId);
} }
return getColor(type, dossierTemplateId); return getColor(type, dossierTemplateId);
@ -324,5 +332,12 @@ public class RedactionLogMergeService {
} }
private String getTypeFromTypeId(String typeId){
return typeId.split(":")[0];
}
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java
View File

@ -30,4 +30,8 @@ public class IdBuilder {
} }
public String getTypeId(String type, String dossierTemplateId, String dossierId) {
return type + ":" + dossierTemplateId + (dossierId != null ? ":" + dossierId : "");
}
} }

141
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,27 +1,27 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import com.amazonaws.services.s3.AmazonS3; import static org.assertj.core.api.Assertions.assertThat;
import com.fasterxml.jackson.databind.ObjectMapper; import static org.mockito.Mockito.when;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Type;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows; import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.After; import org.junit.After;
@ -47,15 +47,46 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import java.io.*; import com.amazonaws.services.s3.AmazonS3;
import java.net.URL; import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.charset.StandardCharsets; import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import java.time.OffsetDateTime; import com.iqser.red.service.persistence.service.v1.api.model.JSONPrimitive;
import java.util.*; import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import java.util.stream.Collectors; import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationId;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageService;
import static org.assertj.core.api.Assertions.assertThat; import lombok.SneakyThrows;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -685,7 +716,7 @@ public class RedactionIntegrationTest {
redactionLog.getRedactionLogEntry().forEach(entry -> { redactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.isImage()) { if (entry.isImage()) {
System.out.println("---->" + entry.getType()); System.out.println("---->" + entry.getTypeId());
} }
}); });
@ -744,9 +775,9 @@ public class RedactionIntegrationTest {
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder() manualRedactions.setImageRecategorization(Set.of(ManualImageRecategorization.builder()
.id("37eee3e9d589a5cc529bfec38c3ba479") .id(new AnnotationId("37eee3e9d589a5cc529bfec38c3ba479", "fileId"))
.status(Status.APPROVED) .status(AnnotationStatus.APPROVED)
.type("signature") .type("signature")
.build())); .build()));
@ -829,13 +860,13 @@ public class RedactionIntegrationTest {
.text("This is a comment test") .text("This is a comment test")
.build(); .build();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id("5b940b2cb401ed9f5be6fc24f6e77bcf") .id(new AnnotationId("5b940b2cb401ed9f5be6fc24f6e77bcf", "fileId"))
.status(Status.DECLINED) .status(AnnotationStatus.DECLINED)
.build())); .build()));
manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder() manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.id("675eba69b0c2917de55462c817adaa05") .id(new AnnotationId("675eba69b0c2917de55462c817adaa05", "fileId"))
.legalBasis("Something") .legalBasis("Something")
.status(Status.APPROVED) .status(AnnotationStatus.APPROVED)
.build())); .build()));
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment)); manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
@ -843,12 +874,26 @@ public class RedactionIntegrationTest {
manualRedactions.getComments().put(manualAddId, List.of(comment)); manualRedactions.getComments().put(manualAddId, List.of(comment));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setId(manualAddId); manualRedactionEntry.setId(new AnnotationId(manualAddId, "fileId"));
manualRedactionEntry.setStatus(Status.REQUESTED); manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED);
manualRedactionEntry.setType("name"); manualRedactionEntry.setTypeId("name:" + TEST_DOSSIER_TEMPLATE_ID);
manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction"); manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), 7.648041f, 43.72262f, 1), new Rectangle(new Point(384.83517f, 241.282f), 7.648041f, 17.043358f, 1))); manualRedactionEntry.setPositions(List.of(Rectangle
.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1).build()
, Rectangle
.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1).build())
);
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry); // manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
@ -859,14 +904,14 @@ public class RedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id("5b940b2cb401ed9f5be6fc24f6e77bcf") .id(new AnnotationId("5b940b2cb401ed9f5be6fc24f6e77bcf", "fileId"))
.status(Status.APPROVED) .status(AnnotationStatus.APPROVED)
.build())); .build()));
manualRedactions.setManualLegalBasisChanges(Set.of(ManualLegalBasisChange.builder() manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.id("675eba69b0c2917de55462c817adaa05") .id(new AnnotationId("675eba69b0c2917de55462c817adaa05", "fileId"))
.legalBasis("Manual Legal Basis Change") .legalBasis("Manual Legal Basis Change")
.status(Status.APPROVED) .status(AnnotationStatus.APPROVED)
.build())); .build())));
analyzeService.reanalyze(request); analyzeService.reanalyze(request);