Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #332: RED-3242 - excluded from automatic analysis - reworked redactionLog architecture

Browse Source 
Merge in RED/redaction-service from RED-3242 to master

* commit 'd92a723fea21280d7bf482c3d74c7b177dbaa030':
  RED-3242 - excluded from automatic analysis - reworked redactionLog architecture
This commit is contained in:
Timo Bejan 2022-01-27 14:06:35 +01:00
commit 0da2a1ab26
15 changed files with 191 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/pom.xml
View File

@ -12,7 +12,7 @@
<artifactId>redaction-service-api-v1</artifactId> <artifactId>redaction-service-api-v1</artifactId>
<properties> <properties>
<persistence-service.version>0.151.0</persistence-service.version> <persistence-service.version>1.23.0</persistence-service.version>
</properties> </properties>
<dependencies> <dependencies>

48
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java Normal file
View File

@ -0,0 +1,48 @@
package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Map;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class ManualChange {
private AnnotationStatus annotationStatus;
private ManualRedactionType manualRedactionType;
private OffsetDateTime processedDate;
private String userId;
private Map<String, Object> propertyChanges = new HashMap<>();
public boolean isProcessed() {
return processedDate != null;
}
public static ManualChange from(BaseAnnotation baseAnnotation) {
ManualChange manualChange = new ManualChange();
manualChange.annotationStatus = baseAnnotation.getStatus();
manualChange.processedDate = baseAnnotation.getProcessedDate();
manualChange.userId = baseAnnotation.getUser();
return manualChange;
}
public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
this.manualRedactionType = manualRedactionType;
return this;
}
public ManualChange withChange(String property, Object value) {
this.propertyChanges.put(property, value);
return this;
}
}

10
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java
View File

@ -1,5 +1,13 @@
package com.iqser.red.service.redaction.v1.model; package com.iqser.red.service.redaction.v1.model;
public enum ManualRedactionType { public enum ManualRedactionType {
ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE ADD_LOCALLY,
ADD_TO_DICTIONARY,
REMOVE_LOCALLY,
REMOVE_FROM_DICTIONARY,
FORCE_REDACT,
FORCE_HINT,
RECATEGORIZE,
LEGAL_BASIS_CHANGE,
RESIZE
} }

50
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -2,16 +2,9 @@ package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import lombok.AllArgsConstructor; import lombok.*;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import java.util.ArrayList; import java.util.*;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Data @Data
@ -28,27 +21,18 @@ public class RedactionLogEntry {
private int matchedRule; private int matchedRule;
private boolean rectangle; private boolean rectangle;
private String legalBasis; private String legalBasis;
private boolean redacted; private boolean redacted;
private boolean isHint; private boolean isHint;
private boolean isRecommendation; private boolean isRecommendation;
private boolean hasBeenResized;
private boolean hasBeenRecategorized;
private boolean hasLegalBasisChanged;
private boolean hasBeenForced;
private boolean hasBeenRemovedByManualOverride;
private String section; private String section;
private float[] color; private float[] color;
@Builder.Default @Builder.Default
private List<Rectangle> positions = new ArrayList<>(); private List<Rectangle> positions = new ArrayList<>();
private int sectionNumber; private int sectionNumber;
private boolean manual;
private AnnotationStatus status;
private ManualRedactionType manualRedactionType;
private String manualRedactionUserId;
private boolean isDictionaryEntry;
private String textBefore; private String textBefore;
private String textAfter; private String textAfter;
@ -62,21 +46,39 @@ public class RedactionLogEntry {
private boolean isImage; private boolean isImage;
private boolean imageHasTransparency; private boolean imageHasTransparency;
private boolean isDictionaryEntry;
private boolean isDossierDictionaryEntry; private boolean isDossierDictionaryEntry;
private boolean excluded; private boolean excluded;
private String recategorizationType;
private String legalBasisChangeValue;
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
@Builder.Default @Builder.Default
private List<Change> changes = new ArrayList<>(); private List<Change> changes = new ArrayList<>();
private Set<Engine> engines= new HashSet<>(); @EqualsAndHashCode.Exclude
@Builder.Default
private List<ManualChange> manualChanges = new ArrayList<>();
private Set<Engine> engines = new HashSet<>();
private Set<String> reference = new HashSet<>(); private Set<String> reference = new HashSet<>();
public boolean lastChangeIsRemoved() {
return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
}
public boolean isLocalManualRedaction() {
return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY &&
mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
}
public boolean isManuallyRemoved() {
return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY &&
mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
}
private <T> Optional<T> last(List<T> list) {
return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
}
} }

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -1,8 +1,11 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@ -180,7 +183,7 @@ public class AnalyzeService {
Set<Integer> sectionsToReanalyse = new HashSet<>(); Set<Integer> sectionsToReanalyse = new HashSet<>();
Map<Integer, Set<Image>> imageEntries = new HashMap<>(); Map<Integer, Set<Image>> imageEntries = new HashMap<>();
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
sectionsToReanalyse.add(entry.getSectionNumber()); sectionsToReanalyse.add(entry.getSectionNumber());
} }
if (entry.isImage()) { if (entry.isImage()) {
@ -231,6 +234,7 @@ public class AnalyzeService {
.legalBasisVersion(redactionLog.getLegalBasisVersion()) .legalBasisVersion(redactionLog.getLegalBasisVersion())
.dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion()) .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
.wasReanalyzed(isReanalysis) .wasReanalyzed(isReanalysis)
.manualRedactions(analyzeRequest.getManualRedactions())
.build(); .build();
} }

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.model.*;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
@ -15,13 +16,10 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.awt.Color; import java.awt.*;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Service @Service
@ -107,7 +105,7 @@ public class AnnotationService {
private String createAnnotationContent(RedactionLogEntry redactionLogEntry) { private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.isManual()) { if (redactionLogEntry.isLocalManualRedaction()) {
return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\""; return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
} }
return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -6,7 +6,6 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import feign.FeignException; import feign.FeignException;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -14,7 +13,8 @@ import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.SerializationUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.awt.Color; import java.awt.*;
import java.util.List;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -63,7 +63,7 @@ public class DictionaryService {
}); });
}); });
if(dictionariesByDossier.containsKey(dossierId)) { if (dictionariesByDossier.containsKey(dossierId)) {
dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary(); dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> { dictionaryModel.getEntries().forEach(dictionaryEntry -> {
@ -105,7 +105,7 @@ public class DictionaryService {
dictionaryRepresentation.setDictionaryVersion(version); dictionaryRepresentation.setDictionaryVersion(version);
dictionaryRepresentation.setDictionary(dictionary); dictionaryRepresentation.setDictionary(dictionary);
if(dossierId == null) { if (dossierId == null) {
dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation); dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
} else { } else {
dictionariesByDossier.put(dossierId, dictionaryRepresentation); dictionariesByDossier.put(dossierId, dictionaryRepresentation);
@ -204,7 +204,7 @@ public class DictionaryService {
//TODO merge dictionaries if they have same names //TODO merge dictionaries if they have same names
long dossierDictionaryVersion = -1; long dossierDictionaryVersion = -1;
if(dictionariesByDossier.containsKey(dossierId)) { if (dictionariesByDossier.containsKey(dossierId)) {
var dossierRepresentation = dictionariesByDossier.get(dossierId); var dossierRepresentation = dictionariesByDossier.get(dossierId);
dossierRepresentation.getDictionary().forEach(dm -> { dossierRepresentation.getDictionary().forEach(dm -> {
copy.add(SerializationUtils.clone(dm)); copy.add(SerializationUtils.clone(dm));

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -1,43 +1,27 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j @Slf4j
@Service @Service
@ -249,7 +233,7 @@ public class EntityRedactionService {
nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellstarts, headline)); nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellstarts, headline));
} }
return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound) ; return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
} }

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -1,16 +1,9 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@ -20,9 +13,14 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@Slf4j @Slf4j
@Service @Service
@ -90,9 +88,9 @@ public class ManualRedactionSurroundingTextService {
Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions); Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);
if(correctEntity == null){ if (correctEntity == null) {
log.warn("Could not calculate surrounding text"); log.warn("Could not calculate surrounding text");
return Pair.of("",""); return Pair.of("", "");
} }
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) { if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java
View File

@ -33,17 +33,11 @@ public class RedactionChangeLogService {
List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry() List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
.stream() .stream()
.filter(entry -> !entry.getChanges() .filter(entry -> !entry.lastChangeIsRemoved())
.get(entry.getChanges().size() - 1)
.getType()
.equals(ChangeType.REMOVED))
.collect(Collectors.toList()); .collect(Collectors.toList());
Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream() Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
.filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges() .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()).collect(Collectors.toSet());
.get(entry.getChanges().size() - 1)
.getType()
.equals(ChangeType.REMOVED)).collect(Collectors.toSet());
notRemovedPreviousEntries.forEach(added::remove); notRemovedPreviousEntries.forEach(added::remove);
@ -87,7 +81,7 @@ public class RedactionChangeLogService {
addedIds.forEach((k, v) -> { addedIds.forEach((k, v) -> {
if (v.getChanges().isEmpty()) { if (v.getChanges().isEmpty()) {
v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now())); v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
} }
newRedactionLogEntries.add(v); newRedactionLogEntries.add(v);
}); });

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -1,15 +1,5 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
@ -20,9 +10,13 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
@Service @Service
@Slf4j @Slf4j
@ -70,7 +64,6 @@ public class RedactionLogCreatorService {
.legalBasis(image.getLegalBasis()) .legalBasis(image.getLegalBasis())
.matchedRule(image.getMatchedRule()) .matchedRule(image.getMatchedRule())
.isHint(dictionaryService.isHint(image.getType(), dossierTemplateId)) .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
.manual(false)
.isDictionaryEntry(false) .isDictionaryEntry(false)
.isRecommendation(false) .isRecommendation(false)
.positions(List.of(new Rectangle(new Point((float) image.getPosition() .positions(List.of(new Rectangle(new Point((float) image.getPosition()
@ -173,10 +166,7 @@ public class RedactionLogCreatorService {
private boolean isCharClose(float x, float xDirAdj, float width) { private boolean isCharClose(float x, float xDirAdj, float width) {
float max = x + (5 * width); float max = x + (5 * width);
if (xDirAdj < max) { return xDirAdj < max;
return true;
}
return false;
} }
@ -190,11 +180,7 @@ public class RedactionLogCreatorService {
float minY = y - offsetY; float minY = y - offsetY;
float maxY = y + offsetY; float maxY = y + offsetY;
if (yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight) { return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight;
return true;
}
return false;
} }

101
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -1,7 +1,9 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.model.*;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
@ -46,7 +48,7 @@ public class RedactionLogMergeService {
if (excludedPages != null && !excludedPages.isEmpty()) { if (excludedPages != null && !excludedPages.isEmpty()) {
entry.getPositions().forEach(pos -> { entry.getPositions().forEach(pos -> {
if (!entry.isManual() && excludedPages.contains(pos.getPage())) { if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
entry.setExcluded(true); entry.setExcluded(true);
} }
}); });
@ -117,9 +119,9 @@ public class RedactionLogMergeService {
var imageRecategorization = (ManualImageRecategorization) mrw.getItem(); var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) { if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setType(imageRecategorization.getType()); redactionLogEntry.setType(imageRecategorization.getType());
redactionLogEntry.setHasBeenRecategorized(true);
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType()); redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) { if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
@ -132,77 +134,65 @@ public class RedactionLogMergeService {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
.withChange("type", imageRecategorization.getType()));
} }
if (mrw.getItem() instanceof IdRemoval) { if (mrw.getItem() instanceof IdRemoval) {
var manualRemoval = (IdRemoval) mrw.getItem(); var manualRemoval = (IdRemoval) mrw.getItem();
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) { if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
log.info("Skipping merge for dictionary-modifying entry"); log.debug("Skipping merge for dictionary-modifying entry");
} else { } else {
String manualOverrideReason = null; String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) { if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setRedacted(false); redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
redactionLogEntry.setHint(false); redactionLogEntry.setHint(false);
redactionLogEntry.setHasBeenRemovedByManualOverride(true);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser()); redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
} }
} }
if (mrw.getItem() instanceof ManualForceRedaction) { if (mrw.getItem() instanceof ManualForceRedaction) {
var manualForceRedact = (ManualForceRedaction) mrw.getItem(); var manualForceRedact = (ManualForceRedaction) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) { if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
// Forcing a skipped hint should result in a hint // Forcing a skipped hint should result in a hint
if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) { if (dictionaryIsHint) {
redactionLogEntry.setHint(true); redactionLogEntry.setHint(true);
} else { } else {
redactionLogEntry.setRedacted(true); redactionLogEntry.setRedacted(true);
} }
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setHasBeenForced(true);
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true); var manualChange = ManualChange.from(manualForceRedact)
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
redactionLogEntry.getManualChanges().add(manualChange);
} }
@ -210,11 +200,9 @@ public class RedactionLogMergeService {
var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem(); var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) { if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
redactionLogEntry.setRedacted(true); redactionLogEntry.setRedacted(true);
redactionLogEntry.setHasLegalBasisChanged(true);
if (manualLegalBasisChange.getSection() != null) { if (manualLegalBasisChange.getSection() != null) {
redactionLogEntry.setSection(manualLegalBasisChange.getSection()); redactionLogEntry.setSection(manualLegalBasisChange.getSection());
} }
@ -223,47 +211,45 @@ public class RedactionLogMergeService {
} }
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); var manualChange = ManualChange.from(manualLegalBasisChange)
.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
if (manualLegalBasisChange.getSection() != null) {
manualChange.withChange("section", manualLegalBasisChange.getSection());
}
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
manualChange.withChange("value", manualLegalBasisChange.getValue());
}
redactionLogEntry.getManualChanges().add(manualChange);
} }
if (mrw.getItem() instanceof ManualResizeRedaction) { if (mrw.getItem() instanceof ManualResizeRedaction) {
var manualResizeRedact = (ManualResizeRedaction) mrw.getItem(); var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
String manualOverrideReason = null; String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) { if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setValue(manualResizeRedact.getValue()); redactionLogEntry.setValue(manualResizeRedact.getValue());
redactionLogEntry.setHasBeenResized(true);
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore()); redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter()); redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact"); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore()); redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter()); redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
} }
redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser());
redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true); redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE); .withManualRedactionType(ManualRedactionType.RESIZE)
.withChange("value", manualResizeRedact.getValue()));
} }
}); });
@ -319,14 +305,17 @@ public class RedactionLogMergeService {
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus() return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) &&
.equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); manualRedactionEntry.getProcessedDate() != null &&
(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
} }
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String dossierTemplateId) { String dossierTemplateId) {
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(id) .id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
@ -340,11 +329,9 @@ public class RedactionLogMergeService {
.redacted(true) .redacted(true)
.isHint(false) .isHint(false)
.sectionNumber(-1) .sectionNumber(-1)
.manual(true)
.status(manualRedactionEntry.getStatus())
.manualRedactionType(ManualRedactionType.ADD)
.manualRedactionUserId(manualRedactionEntry.getUser())
.rectangle(manualRedactionEntry.isRectangle()) .rectangle(manualRedactionEntry.isRectangle())
.manualChanges(List.of(ManualChange.from(manualRedactionEntry)
.withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)))
.build(); .build();
} }

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.List;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.CellRectangle; import com.iqser.red.service.redaction.v1.model.CellRectangle;
import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.SectionRectangle; import com.iqser.red.service.redaction.v1.model.SectionRectangle;
@ -14,8 +9,11 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java
View File

@ -1,35 +1,19 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.Footer; import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@Slf4j @Slf4j
@Service @Service

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -23,6 +23,7 @@ import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
@ -51,13 +52,7 @@ import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;