Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 deiflaender
|
0c8ff214e4 |
@@ -37,7 +37,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.14.0</version>
|
||||
<version>1.13.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<persistence-service.version>1.240.0</persistence-service.version>
|
||||
<persistence-service.version>1.166.0</persistence-service.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SimplifiedSectionText {
|
||||
|
||||
private int sectionNumber;
|
||||
private String text;
|
||||
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SimplifiedText {
|
||||
|
||||
private int numberOfPages;
|
||||
private List<SimplifiedSectionText> simplifiedSectionTexts = new ArrayList<>();
|
||||
|
||||
}
|
||||
+6
-18
@@ -301,24 +301,12 @@ class LegacyPDFStreamEngine extends PDFStreamEngine
|
||||
nextY -= pageSize.getLowerLeftY();
|
||||
}
|
||||
|
||||
// This is a hack for unicode letter with 2 chars e.g. RA see unicodeProblem.pdf
|
||||
if(unicodeMapping.length() == 2){
|
||||
processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
|
||||
pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
|
||||
Math.abs(dyDisplay), dxDisplay,
|
||||
Math.abs(spaceWidthDisplay), Character.toString(unicodeMapping.charAt(0)), new int[] { code }, font,
|
||||
fontSize,
|
||||
(int)(fontSize * textMatrix.getScalingFactorX())));
|
||||
processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
|
||||
pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
|
||||
Math.abs(dyDisplay), dxDisplay,
|
||||
Math.abs(spaceWidthDisplay), Character.toString(unicodeMapping.charAt(1)), new int[] { code }, font,
|
||||
fontSize,
|
||||
(int)(fontSize * textMatrix.getScalingFactorX())));
|
||||
} else {
|
||||
|
||||
processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(), pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY, Math.abs(dyDisplay), dxDisplay, Math.abs(spaceWidthDisplay), unicodeMapping, new int[]{code}, font, fontSize, (int) (fontSize * textMatrix.getScalingFactorX())));
|
||||
}
|
||||
processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
|
||||
pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
|
||||
Math.abs(dyDisplay), dxDisplay,
|
||||
Math.abs(spaceWidthDisplay), unicodeMapping, new int[] { code }, font,
|
||||
fontSize,
|
||||
(int)(fontSize * textMatrix.getScalingFactorX())));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+12
-49
@@ -1,21 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
@@ -24,31 +8,15 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.entity
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
|
||||
@@ -60,6 +28,15 @@ import io.micrometer.core.annotation.Timed;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -116,9 +93,8 @@ public class AnalyzeService {
|
||||
.map(SectionArea::getPage)
|
||||
.collect(Collectors.toSet()), sectionText.getSectionAreas())));
|
||||
|
||||
log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
@@ -336,17 +312,4 @@ public class AnalyzeService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private SimplifiedText convert(Text text) {
|
||||
|
||||
List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
|
||||
int numberOfPages = 0;
|
||||
|
||||
if (text != null) {
|
||||
numberOfPages = text.getNumberOfPages();
|
||||
text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
|
||||
}
|
||||
return SimplifiedText.builder().numberOfPages(numberOfPages).simplifiedSectionTexts(sectionTexts).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+6
-5
@@ -5,10 +5,12 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
|
||||
import feign.FeignException;
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -23,7 +25,6 @@ import java.util.stream.Collectors;
|
||||
@RequiredArgsConstructor
|
||||
public class DictionaryService {
|
||||
|
||||
private static final String DEFAULT_COLOR = "#cccccc";
|
||||
private final DictionaryClient dictionaryClient;
|
||||
|
||||
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
|
||||
@@ -177,10 +178,10 @@ public class DictionaryService {
|
||||
|
||||
Colors colors = dictionaryClient.getColors(dossierTemplateId);
|
||||
|
||||
dictionaryRepresentation.setDefaultColor(convertColor(DEFAULT_COLOR));
|
||||
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAddColor()));
|
||||
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemoveColor()));
|
||||
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getSkippedColor()));
|
||||
dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
|
||||
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
|
||||
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
|
||||
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
|
||||
dictionaryRepresentation.setDossierTemplateId(dossierTemplateId);
|
||||
dictionaryRepresentation.setDictionaryVersion(version);
|
||||
dictionaryRepresentation.setDictionary(dictionary);
|
||||
|
||||
+1
-2
@@ -115,7 +115,7 @@ public class ImportedRedactionService {
|
||||
Point p = new Point();
|
||||
|
||||
if (rectangle.getWidth() < 0) {
|
||||
p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
|
||||
p.setX(rectangle.getTopLeft().getX() - rectangle.getWidth());
|
||||
r.setWidth(Math.abs(rectangle.getWidth()));
|
||||
} else {
|
||||
p.setX(rectangle.getTopLeft().getX());
|
||||
@@ -127,7 +127,6 @@ public class ImportedRedactionService {
|
||||
r.setHeight(Math.abs(rectangle.getHeight()));
|
||||
} else {
|
||||
p.setY(rectangle.getTopLeft().getY());
|
||||
r.setHeight(rectangle.getHeight());
|
||||
}
|
||||
|
||||
r.setTopLeft(p);
|
||||
|
||||
+44
-86
@@ -1,40 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualChange;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
@@ -43,13 +18,20 @@ import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.*;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogMergeService {
|
||||
|
||||
private static final String DELETED_TYPE_COLOR = "#9398a0";
|
||||
|
||||
private final SectionTextService sectionTextService;
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
@@ -57,7 +39,6 @@ public class RedactionLogMergeService {
|
||||
|
||||
@Timed("redactmanager_getMergedRedactionLog")
|
||||
public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
|
||||
|
||||
log.debug("Requested preview for: {}", redactionRequest);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
@@ -91,15 +72,15 @@ public class RedactionLogMergeService {
|
||||
return merged;
|
||||
}
|
||||
|
||||
|
||||
private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
|
||||
ManualRedactions manualRedactions, Set<Integer> excludedPages,
|
||||
List<Type> types, Colors colors) {
|
||||
private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
|
||||
Set<Integer> excludedPages, List<Type> types, Colors colors) {
|
||||
|
||||
var skippedImportedRedactions = new HashSet<>();
|
||||
|
||||
log.info("Merging Redaction log with manual redactions");
|
||||
if (manualRedactions != null) {
|
||||
|
||||
|
||||
var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
|
||||
|
||||
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
|
||||
@@ -131,12 +112,9 @@ public class RedactionLogMergeService {
|
||||
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
redactionLog.getRedactionLogEntry().removeIf(entry -> {
|
||||
|
||||
if (entry.getImportedRedactionIntersections() != null) {
|
||||
entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
|
||||
if (!entry.getImportedRedactionIntersections()
|
||||
.isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType()
|
||||
.equals("image") || entry.getType().equals("ocr")))) {
|
||||
if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -201,6 +179,7 @@ public class RedactionLogMergeService {
|
||||
String manualOverrideReason = null;
|
||||
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
|
||||
|
||||
redactionLogEntry.setType(imageRecategorization.getType());
|
||||
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
|
||||
|
||||
@@ -217,20 +196,16 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
|
||||
}
|
||||
|
||||
if(manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(imageRecategorization)
|
||||
.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.withChange("type", imageRecategorization.getType()));
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
|
||||
.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.withChange("type", imageRecategorization.getType()));
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof IdRemoval) {
|
||||
var manualRemoval = (IdRemoval) mrw.getItem();
|
||||
if (manualRemoval.getStatus()
|
||||
.equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
|
||||
log.debug("Skipping merge for dictionary-modifying entry");
|
||||
} else {
|
||||
String manualOverrideReason = null;
|
||||
@@ -244,13 +219,11 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
|
||||
}
|
||||
|
||||
if(manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
|
||||
ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -274,10 +247,7 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
}
|
||||
|
||||
if(manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
var manualChange = ManualChange.from(manualForceRedact)
|
||||
.withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
|
||||
|
||||
@@ -303,9 +273,7 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
|
||||
}
|
||||
|
||||
if(manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
var manualChange = ManualChange.from(manualLegalBasisChange)
|
||||
.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
|
||||
@@ -328,7 +296,7 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setValue(manualResizeRedact.getValue());
|
||||
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
@@ -339,17 +307,16 @@ public class RedactionLogMergeService {
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualResizeRedact)
|
||||
.withManualRedactionType(ManualRedactionType.RESIZE)
|
||||
.withChange("value", manualResizeRedact.getValue()));
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
|
||||
.withManualRedactionType(ManualRedactionType.RESIZE)
|
||||
.withChange("value", manualResizeRedact.getValue()));
|
||||
}
|
||||
|
||||
});
|
||||
@@ -371,8 +338,7 @@ public class RedactionLogMergeService {
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
|
||||
Map<String, List<Comment>> comments, Colors colors,
|
||||
List<Type> types) {
|
||||
Map<String, List<Comment>> comments, Colors colors, List<Type> types) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
|
||||
|
||||
@@ -394,12 +360,9 @@ public class RedactionLogMergeService {
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
|
||||
private List<RedactionLogComment> convert(List<Comment> comments) {
|
||||
|
||||
return comments == null ? null : comments.stream()
|
||||
.map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
|
||||
.collect(Collectors.toList());
|
||||
return comments == null ? null :comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(),
|
||||
c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@@ -414,8 +377,8 @@ public class RedactionLogMergeService {
|
||||
|
||||
@SuppressWarnings("PMD.UselessParentheses")
|
||||
private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
|
||||
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
|
||||
|| ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
|
||||
}
|
||||
|
||||
|
||||
@@ -449,14 +412,14 @@ public class RedactionLogMergeService {
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped,
|
||||
List<Type> types) {
|
||||
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
|
||||
boolean skipped, List<Type> types) {
|
||||
|
||||
if (requested) {
|
||||
return convertColor(colors.getRequestRemoveColor());
|
||||
return convertColor(colors.getRequestRemove());
|
||||
}
|
||||
if (skipped || !isRedaction && !isHint(types, type)) {
|
||||
return convertColor(colors.getSkippedColor());
|
||||
return convertColor(colors.getNotRedacted());
|
||||
}
|
||||
return getColor(types, type);
|
||||
}
|
||||
@@ -465,9 +428,9 @@ public class RedactionLogMergeService {
|
||||
private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
|
||||
|
||||
if (status.equals(AnnotationStatus.REQUESTED)) {
|
||||
return convertColor(colors.getRequestAddColor());
|
||||
return convertColor(colors.getRequestAdd());
|
||||
} else if (status.equals(AnnotationStatus.DECLINED)) {
|
||||
return convertColor(colors.getSkippedColor());
|
||||
return convertColor(colors.getNotRedacted());
|
||||
}
|
||||
return getColor(types, type);
|
||||
}
|
||||
@@ -481,8 +444,7 @@ public class RedactionLogMergeService {
|
||||
return convertColor(foundAndNotDeletedType.get().getHexColor());
|
||||
}
|
||||
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
|
||||
return firstDeletedType.map(value -> convertColor(value.getHexColor()))
|
||||
.orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
|
||||
return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
|
||||
}
|
||||
|
||||
|
||||
@@ -504,15 +466,11 @@ public class RedactionLogMergeService {
|
||||
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
|
||||
}
|
||||
|
||||
|
||||
private List<Type> getMatchingTypes(List<Type> types, String type) {
|
||||
|
||||
return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private boolean isDeletedType(Type type) {
|
||||
|
||||
return type.getSoftDeletedTime() != null;
|
||||
}
|
||||
|
||||
|
||||
+1
-7
@@ -64,16 +64,10 @@ public class RedactionStorageService {
|
||||
@Timed("redactmanager_serializeObject")
|
||||
private byte[] serializeObject(Object any) {
|
||||
|
||||
try {
|
||||
var baos = new ByteArrayOutputStream();
|
||||
dslJson.serialize(any, baos);
|
||||
return baos.toByteArray();
|
||||
} catch (com.dslplatform.json.SerializationException e) {
|
||||
// Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
var bytes = objectMapper.writeValueAsBytes(any);
|
||||
dslJson.newWriter();
|
||||
return bytes;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ public class DictionaryServiceTest {
|
||||
|
||||
when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
|
||||
when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", "#cccccc",
|
||||
"#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc","#cccccc",
|
||||
"#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc",
|
||||
"#cccccc", "#cccccc", "#cccccc", "#cccccc"));
|
||||
|
||||
var type1 = new Type();
|
||||
|
||||
+4
-27
@@ -807,30 +807,6 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testUnicodeProblem() throws IOException {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
AnalyzeRequest request = prepareStorage("files/new/unicodeProblem.pdf");
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
System.out.println("numberOfPages: " + result.getNumberOfPages());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testRotations() throws IOException {
|
||||
|
||||
@@ -1617,9 +1593,10 @@ public class RedactionIntegrationTest {
|
||||
rankTypeMap.put(DOSSIER_REDACTIONS, 200);
|
||||
rankTypeMap.put(IMPORTED_REDACTION, 200);
|
||||
|
||||
colors.setSkippedColor("#cccccc");
|
||||
colors.setRequestAddColor("#04b093");
|
||||
colors.setRequestRemoveColor("#04b093");
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
colors.setRequestAdd("#04b093");
|
||||
colors.setRequestRemove("#04b093");
|
||||
}
|
||||
|
||||
|
||||
|
||||
+7
-8
@@ -1,14 +1,13 @@
|
||||
{
|
||||
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
|
||||
"requestAddColor": "#04b093",
|
||||
"requestRemoveColor": "#04b093",
|
||||
"defaultColor": "#9398a0",
|
||||
"requestAdd": "#04b093",
|
||||
"requestRemove": "#04b093",
|
||||
"notRedacted": "#c498fa",
|
||||
"analysisColor": "#dd4d50",
|
||||
"updatedColor": "#fdbd00",
|
||||
"dictionaryRequestColor": "#5b97db",
|
||||
"manualRedactionColor": "#9398a0",
|
||||
"previewColor": "#9398a0",
|
||||
"ignoredHintColor": "#e7d4ff",
|
||||
"recommendationColor": "#cccccc",
|
||||
"hintColor": "#cccccc",
|
||||
"redactionColor": "#cccccc",
|
||||
"skippedColor": "#cccccc"
|
||||
}
|
||||
"ignoredHintColor": "#e7d4ff"
|
||||
}
|
||||
+1
-2
@@ -12,5 +12,4 @@ Xinyi Y. Tao
|
||||
Dorn
|
||||
Prasher
|
||||
David
|
||||
annotation
|
||||
J.B. RASCLE
|
||||
annotation
|
||||
BIN
Binary file not shown.
+6
-8
@@ -1,14 +1,12 @@
|
||||
{
|
||||
"dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
|
||||
"requestAddColor": "#04b093",
|
||||
"requestRemoveColor": "#04b093",
|
||||
"defaultColor": "#9398a0",
|
||||
"requestAdd": "#04b093",
|
||||
"requestRemove": "#04b093",
|
||||
"notRedacted": "#c498fa",
|
||||
"analysisColor": "#dd4d50",
|
||||
"updatedColor": "#fdbd00",
|
||||
"dictionaryRequestColor": "#5b97db",
|
||||
"manualRedactionColor": "#9398a0",
|
||||
"previewColor": "#9398a0",
|
||||
"ignoredHintColor": "#e7d4ff",
|
||||
"recommendationColor": "#cccccc",
|
||||
"hintColor": "#cccccc",
|
||||
"redactionColor": "#cccccc",
|
||||
"skippedColor": "#cccccc"
|
||||
"ignoredHintColor": "#e7d4ff"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user