Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-2228: Use manual redaction classes from persistence-service, changed type in redactionlog to typeId

Browse Source
This commit is contained in:
Dominique Eifländer 2021-09-23 12:15:00 +02:00
parent d9b78643fb
commit 6ca134bdbd
18 changed files with 202 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -11,6 +11,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
@Data
@Builder
@NoArgsConstructor

21
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Comment.java
View File

@ -1,21 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class Comment {
private String id;
private OffsetDateTime date;
private String text;
private String user;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/IdRemoval.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class IdRemoval {
private String id;
private String user;
private Status status;
private boolean removeFromDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualForceRedact {
private String id;
private String user;
private Status status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualImageRecategorization.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualImageRecategorization {
private String id;
private String user;
private Status status;
private String type;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

25
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java
View File

@ -1,25 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualLegalBasisChange {
private String id;
private String user;
private Status status;
private String legalBasis;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

34
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java
View File

@ -1,34 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualRedactionEntry {
private String id;
private String user;
private String type;
private String value;
private String reason;
private String legalBasis;
private List<Rectangle> positions = new ArrayList<>();
private Status status;
private boolean addToDictionary;
private boolean addToDossierDictionary;
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;
}

38
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java
View File

@ -1,38 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualRedactions {
@Builder.Default
private Set<IdRemoval> idsToRemove = new HashSet<>();
@Builder.Default
private Set<ManualForceRedact> forceRedacts = new HashSet<>();
@Builder.Default
private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
@Builder.Default
private Set<ManualImageRecategorization> imageRecategorizations = new HashSet<>();
@Builder.Default
private Set<ManualLegalBasisChange> manualLegalBasisChanges = new HashSet<>();
@Builder.Default
private Map<String, List<Comment>> comments = new HashMap<>();
}

8
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -11,6 +11,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
@Data
@Builder
@NoArgsConstructor
@ -19,7 +21,7 @@ import java.util.Set;
public class RedactionLogEntry {
private String id;
private String type;
private String typeId;
private String value;
private String reason;
private int matchedRule;
@ -34,7 +36,7 @@ public class RedactionLogEntry {
private List<Rectangle> positions = new ArrayList<>();
private int sectionNumber;
private boolean manual;
private Status status;
private AnnotationStatus status;
private ManualRedactionType manualRedactionType;
private String manualRedactionUserId;
private boolean isDictionaryEntry;
@ -43,7 +45,7 @@ public class RedactionLogEntry {
private String textAfter;
@Builder.Default
private List<Comment> comments = new ArrayList<>();
private List<com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment> comments = new ArrayList<>();
private int startOffset;
private int endOffset;

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.model;
import java.util.HashSet;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;

5
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Status.java
View File

@ -1,5 +0,0 @@
package com.iqser.red.service.redaction.v1.model;
public enum Status {
REQUESTED, APPROVED, DECLINED
}

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -13,13 +13,14 @@ import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationId;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
@ -113,7 +114,7 @@ public class AnalyzeService {
dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId());
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
.getDossierTemplateId());
.getDossierTemplateId(), analyzeRequest.getDossierId());
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary
@ -166,7 +167,7 @@ public class AnalyzeService {
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
.getDossierTemplateId());
.getDossierTemplateId(), analyzeRequest.getDossierId());
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
@ -186,7 +187,7 @@ public class AnalyzeService {
if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
sectionsToReanalyse.add(entry.getSectionNumber());
}
if (entry.isImage() || entry.getType().equals("image")) {
if (entry.isImage()) {
imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
}
}
@ -242,13 +243,13 @@ public class AnalyzeService {
return new HashSet<>();
}
return Stream.concat(manualRedactions.getManualLegalBasisChanges()
return Stream.concat(manualRedactions.getLegalBasisChanges()
.stream()
.map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations()
.map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorization()
.stream()
.map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove()
.stream()
.map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))))
.map(IdRemoval::getId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getId)))).map(AnnotationId::getId)
.collect(Collectors.toSet());
}
@ -258,7 +259,7 @@ public class AnalyzeService {
Rectangle position = entry.getPositions().get(0);
return Image.builder()
.type(entry.getType())
.type(entry.getTypeId().split(":")[0])
.position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft()
.getY(), position.getWidth(), position.getHeight()))
.sectionNumber(entry.getSectionNumber())

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java
View File

@ -12,6 +12,7 @@ import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import org.springframework.stereotype.Service;
import java.awt.Color;
@ -91,7 +92,7 @@ public class AnnotationService {
if (redactionLogEntry.getComments() != null) {
for (Comment comment : redactionLogEntry.getComments()) {
PDAnnotationText txtAnnot = new PDAnnotationText();
txtAnnot.setAnnotationName(comment.getId());
txtAnnot.setAnnotationName(String.valueOf(comment.getId()));
txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -14,10 +14,10 @@ import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.Status;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
@ -84,12 +84,12 @@ public class EntityRedactionService {
if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
.getImageRecategorizations() != null) {
.getImageRecategorization() != null) {
for (Image image : reanalysisSection.getImages()) {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
.getImageRecategorizations()) {
if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId()
.getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getId()
.equals(imageId)) {
image.setType(imageRecategorization.getType());
}

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -31,13 +31,13 @@ public class RedactionLogCreatorService {
public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages,
String dossierTemplateId) {
String dossierTemplateId, String dossierId) {
List<RedactionLogEntry> entries = new ArrayList<>();
for (int page = 1; page <= numberOfPages; page++) {
if (pageEntities.getEntitiesPerPage().get(page) != null) {
entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId));
entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId, dossierId));
}
if (pageEntities.getImagesPerPage().get(page) != null) {
@ -62,7 +62,7 @@ public class RedactionLogCreatorService {
.id(id)
.color(getColor(image.getType(), dossierTemplateId, image.isRedaction()))
.isImage(true)
.type(image.getType())
.typeId(IdBuilder.getTypeId(image.getType(), dossierTemplateId, null))
.redacted(image.isRedaction())
.reason(image.getRedactionReason())
.legalBasis(image.getLegalBasis())
@ -86,7 +86,7 @@ public class RedactionLogCreatorService {
}
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId) {
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId, String dossierId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -98,7 +98,7 @@ public class RedactionLogCreatorService {
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId, dossierId);
if (processedIds.contains(entityPositionSequence.getId())) {
// TODO refactor this outer loop jump as soon as we have the time.
@ -157,7 +157,7 @@ public class RedactionLogCreatorService {
}
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId, String dossierId) {
Set<String> referenceIds = new HashSet<>();
entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId())));
@ -167,7 +167,7 @@ public class RedactionLogCreatorService {
.reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis())
.value(entity.getWord())
.type(entity.getType())
.typeId(IdBuilder.getTypeId(entity.getType(), dossierTemplateId, entity.isDossierDictionaryEntry() ? dossierId : null))
.redacted(entity.isRedaction())
.isHint(isHint(entity.getType(), dossierTemplateId))
.isRecommendation(isRecommendation(entity.getType(), dossierTemplateId))
@ -206,4 +206,6 @@ public class RedactionLogCreatorService {
return dictionaryService.isRecommendation(type, dossierTemplateId);
}
}

119
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -10,17 +10,20 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.Status;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.AllArgsConstructor;
import lombok.Data;
@ -74,27 +77,27 @@ public class RedactionLogMergeService {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
manualRedactions.getImageRecategorizations().forEach(item -> {
manualRedactions.getImageRecategorization().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
}
});
manualRedactions.getIdsToRemove().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
}
});
manualRedactions.getForceRedacts().forEach(item -> {
manualRedactions.getForceRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
}
});
manualRedactions.getManualLegalBasisChanges().forEach(item -> {
manualRedactions.getLegalBasisChanges().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId(), item.getRequestDate(), item));
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item));
}
});
@ -112,18 +115,18 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof ManualImageRecategorization) {
var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setType(imageRecategorization.getType());
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setTypeId(IdBuilder.getTypeId(imageRecategorization.getType(), dossierTemplateId, null));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) {
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
@ -135,19 +138,19 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof IdRemoval) {
var manualRemoval = (IdRemoval) mrw.getItem();
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry
.isRedacted(), true));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
redactionLogEntry.setReason(manualOverrideReason);
@ -158,24 +161,24 @@ public class RedactionLogMergeService {
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
}
if (mrw.getItem() instanceof ManualForceRedact) {
var manualForceRedact = (ManualForceRedact) mrw.getItem();
if (mrw.getItem() instanceof ManualForceRedaction) {
var manualForceRedact = (ManualForceRedaction) mrw.getItem();
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
@ -188,18 +191,18 @@ public class RedactionLogMergeService {
if (mrw.getItem() instanceof ManualLegalBasisChange) {
var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(getTypeFromTypeId(redactionLogEntry.getTypeId()), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
@ -235,8 +238,8 @@ public class RedactionLogMergeService {
if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry
.getId(), dossierTemplateId);
redactionLogEntry.setPositions(manualRedactionEntry.getPositions());
.getId().getId(), dossierTemplateId);
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId()));
redactionLogEntries.add(redactionLogEntry);
}
@ -245,11 +248,16 @@ public class RedactionLogMergeService {
return redactionLogEntries;
}
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Rectangle> positions){
return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
.getHeight(), pos.getPage())).collect(Collectors.toList());
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus()
.equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
.equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
}
@ -258,13 +266,13 @@ public class RedactionLogMergeService {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.color(getColorForManualAdd(getTypeFromTypeId(manualRedactionEntry.getTypeId()), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType())
.typeId(manualRedactionEntry.getTypeId())
.redacted(true)
.isHint(false)
.section(null)
@ -290,11 +298,11 @@ public class RedactionLogMergeService {
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
if (status.equals(Status.REQUESTED)) {
if (status.equals(AnnotationStatus.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) {
} else if (status.equals(AnnotationStatus.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return getColor(type, dossierTemplateId);
@ -324,5 +332,12 @@ public class RedactionLogMergeService {
}
private String getTypeFromTypeId(String typeId){
return typeId.split(":")[0];
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java
View File

@ -30,4 +30,8 @@ public class IdBuilder {
}
public String getTypeId(String type, String dossierTemplateId, String dossierId) {
return type + ":" + dossierTemplateId + (dossierId != null ? ":" + dossierId : "");
}
}

141
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,27 +1,27 @@
package com.iqser.red.service.redaction.v1.server;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Type;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageService;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import lombok.SneakyThrows;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.junit.After;
@ -47,15 +47,46 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.*;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.stream.Collectors;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationId;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageService;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -685,7 +716,7 @@ public class RedactionIntegrationTest {
redactionLog.getRedactionLogEntry().forEach(entry -> {
if (entry.isImage()) {
System.out.println("---->" + entry.getType());
System.out.println("---->" + entry.getTypeId());
}
});
@ -744,9 +775,9 @@ public class RedactionIntegrationTest {
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder()
.id("37eee3e9d589a5cc529bfec38c3ba479")
.status(Status.APPROVED)
manualRedactions.setImageRecategorization(Set.of(ManualImageRecategorization.builder()
.id(new AnnotationId("37eee3e9d589a5cc529bfec38c3ba479", "fileId"))
.status(AnnotationStatus.APPROVED)
.type("signature")
.build()));
@ -829,13 +860,13 @@ public class RedactionIntegrationTest {
.text("This is a comment test")
.build();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id("5b940b2cb401ed9f5be6fc24f6e77bcf")
.status(Status.DECLINED)
.id(new AnnotationId("5b940b2cb401ed9f5be6fc24f6e77bcf", "fileId"))
.status(AnnotationStatus.DECLINED)
.build()));
manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder()
.id("675eba69b0c2917de55462c817adaa05")
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.id(new AnnotationId("675eba69b0c2917de55462c817adaa05", "fileId"))
.legalBasis("Something")
.status(Status.APPROVED)
.status(AnnotationStatus.APPROVED)
.build()));
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
@ -843,12 +874,26 @@ public class RedactionIntegrationTest {
manualRedactions.getComments().put(manualAddId, List.of(comment));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setId(manualAddId);
manualRedactionEntry.setStatus(Status.REQUESTED);
manualRedactionEntry.setType("name");
manualRedactionEntry.setId(new AnnotationId(manualAddId, "fileId"));
manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED);
manualRedactionEntry.setTypeId("name:" + TEST_DOSSIER_TEMPLATE_ID);
manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), 7.648041f, 43.72262f, 1), new Rectangle(new Point(384.83517f, 241.282f), 7.648041f, 17.043358f, 1)));
manualRedactionEntry.setPositions(List.of(Rectangle
.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1).build()
, Rectangle
.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1).build())
);
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
@ -859,14 +904,14 @@ public class RedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id("5b940b2cb401ed9f5be6fc24f6e77bcf")
.status(Status.APPROVED)
.id(new AnnotationId("5b940b2cb401ed9f5be6fc24f6e77bcf", "fileId"))
.status(AnnotationStatus.APPROVED)
.build()));
manualRedactions.setManualLegalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.id("675eba69b0c2917de55462c817adaa05")
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.id(new AnnotationId("675eba69b0c2917de55462c817adaa05", "fileId"))
.legalBasis("Manual Legal Basis Change")
.status(Status.APPROVED)
.build()));
.status(AnnotationStatus.APPROVED)
.build())));
analyzeService.reanalyze(request);