Compare commits
25
Commits
master_30
...
RED-3553_2
| Author | SHA1 | Date | |
|---|---|---|---|
 Timo Bejan
|
1d1de6caba | ||
|
Timo Bejan
|
7b2295393c | ||
|
Timo Bejan
|
d0ba3f51c1 | ||
|
Timo Bejan
|
c432c56a05 | ||
|
Timo Bejan
|
e73da002d5 | ||
|
Philipp Schramm
|
0ceb8ae266 | ||
|
Philipp Schramm
|
97fe7383cd | ||
|
Dominique Eiflaender
|
7d132cbb88 | ||
|
deiflaender
|
d18b5dfd13 | ||
|
Dominique Eiflaender
|
e2451a08f4 | ||
|
deiflaender
|
537ccb83d7 | ||
|
Ali Oezyetimoglu
|
fd027efa5c | ||
|
aoezyetimoglu
|
c3984e0544 | ||
|
Timo Bejan
|
77feedcc8e | ||
|
Timo Bejan
|
de92117b35 | ||
|
Philipp Schramm
|
143f4cfb0d | ||
|
Philipp Schramm
|
68727178e7 | ||
|
Dominique Eiflaender
|
90fe282313 | ||
|
deiflaender
|
8689125ed9 | ||
|
Dominique Eiflaender
|
e68c6798be | ||
|
deiflaender
|
58b1db66cc | ||
|
deiflaender
|
0072d38d68 | ||
|
deiflaender
|
25a409d3a5 | ||
|
Timo Bejan
|
ca1da20ba3 | ||
|
Timo Bejan
|
3c691a1afc |
@@ -1,4 +1,4 @@
|
||||
FROM red/base-image:1.0.0
|
||||
FROM red/redaction-service-base-v1:1.0.0
|
||||
|
||||
ARG PLATFORM_JAR
|
||||
|
||||
@@ -7,12 +7,3 @@ ENV PLATFORM_JAR ${PLATFORM_JAR}
|
||||
ENV USES_ELASTICSEARCH false
|
||||
|
||||
COPY ["${PLATFORM_JAR}", "/"]
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
wget cabextract xfonts-utils fonts-liberation \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb -o /tmp/ttf-mscorefonts-installer_3.8_all.deb \
|
||||
&& dpkg -i /tmp/ttf-mscorefonts-installer_3.8_all.deb \
|
||||
&& rm /tmp/ttf-mscorefonts-installer_3.8_all.deb \
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.7.0</version>
|
||||
<version>1.11.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<persistence-service.version>1.23.0</persistence-service.version>
|
||||
<persistence-service.version>1.85.0</persistence-service.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
+7
-11
@@ -1,19 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@@ -37,8 +36,5 @@ public class AnalyzeRequest {
|
||||
@Builder.Default
|
||||
private List<FileAttribute> fileAttributes = new ArrayList<>();
|
||||
|
||||
@Builder.Default
|
||||
List<ImportedAnnotation> importedAnnotations = new ArrayList<>();
|
||||
|
||||
}
|
||||
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImportedRedaction {
|
||||
|
||||
private String id;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImportedRedactions {
|
||||
|
||||
@Builder.Default
|
||||
private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
|
||||
}
|
||||
+1
-1
@@ -2,6 +2,6 @@ package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum MessageType {
|
||||
|
||||
FULL_ANALYSE, REANALYSE, SURROUNDING_TEXT
|
||||
ANALYSE, REANALYSE, STRUCTURE_ANALYSE, SURROUNDING_TEXT
|
||||
|
||||
}
|
||||
|
||||
+2
@@ -22,6 +22,8 @@ public class RedactionLogEntry {
|
||||
private boolean rectangle;
|
||||
private String legalBasis;
|
||||
|
||||
private boolean imported;
|
||||
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
|
||||
+43
-42
@@ -1,30 +1,42 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.*;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.io.MemoryUsageSetting;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.stream.Collectors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@@ -43,8 +55,7 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest
|
||||
.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
|
||||
var mergedRedactionLog = getRedactionLog(RedactionRequest.builder()
|
||||
.fileId(annotateRequest.getFileId())
|
||||
.manualRedactions(annotateRequest.getManualRedactions())
|
||||
@@ -73,13 +84,11 @@ public class RedactionController implements RedactionResource {
|
||||
@Override
|
||||
public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
@@ -101,13 +110,11 @@ public class RedactionController implements RedactionResource {
|
||||
@Override
|
||||
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
@@ -131,8 +138,7 @@ public class RedactionController implements RedactionResource {
|
||||
Document classifiedDoc;
|
||||
|
||||
try {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
@@ -163,7 +169,7 @@ public class RedactionController implements RedactionResource {
|
||||
@Override
|
||||
public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
|
||||
|
||||
log.info("Requested preview for: {}", redactionRequest);
|
||||
log.debug("Requested preview for: {}", redactionRequest);
|
||||
dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
@@ -174,8 +180,7 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
|
||||
|
||||
SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest
|
||||
.getFileId());
|
||||
SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
|
||||
if (sectionGrid.getSections().isEmpty()) {
|
||||
|
||||
log.info("SectionGrid does not have headlines set. Computing headlines now!");
|
||||
@@ -184,8 +189,7 @@ public class RedactionController implements RedactionResource {
|
||||
// enhance section grid with headline data
|
||||
for (var sectionText : text.getSectionTexts()) {
|
||||
sectionGrid.getSections()
|
||||
.add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText
|
||||
.getSectionAreas()
|
||||
.add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
|
||||
.stream()
|
||||
.map(SectionArea::getPage)
|
||||
.collect(Collectors.toSet()), sectionText.getSectionAreas()));
|
||||
@@ -194,8 +198,7 @@ public class RedactionController implements RedactionResource {
|
||||
}
|
||||
|
||||
log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
|
||||
return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest
|
||||
.getManualRedactions(), redactionRequest.getExcludedPages());
|
||||
return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
|
||||
}
|
||||
|
||||
|
||||
@@ -217,11 +220,9 @@ public class RedactionController implements RedactionResource {
|
||||
@PathVariable("fileId") String fileId,
|
||||
@RequestBody ManualRedactions manualRedactions) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
|
||||
log.info("add surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, System
|
||||
.currentTimeMillis() - start);
|
||||
return result;
|
||||
log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
|
||||
return result.getManualRedactions();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+19
-26
@@ -1,5 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
@@ -8,18 +15,10 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.NerAnalyserService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -28,7 +27,6 @@ public class RedactionMessageReceiver {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AnalyzeService analyzeService;
|
||||
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
|
||||
private final NerAnalyserService nerAnalyserService;
|
||||
private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
|
||||
|
||||
|
||||
@@ -40,31 +38,26 @@ public class RedactionMessageReceiver {
|
||||
log.info("Processing analyze request for file: {}", analyzeRequest.getFileId());
|
||||
AnalyzeResult result = null;
|
||||
|
||||
switch (analyzeRequest.getMessageType()){
|
||||
switch (analyzeRequest.getMessageType()) {
|
||||
|
||||
case REANALYSE:
|
||||
result = analyzeService.reanalyze(analyzeRequest);
|
||||
log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
case FULL_ANALYSE:
|
||||
// TODO Seperate stucture analysis by other queue
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
|
||||
// TODO NerEntities should be computed and stored in entity-recognition-service, should be triggered by a seperate queue after structure analysis
|
||||
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
|
||||
result = analyzeService.analyze(analyzeRequest);
|
||||
log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result
|
||||
.getDuration());
|
||||
|
||||
case STRUCTURE_ANALYSE:
|
||||
result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
|
||||
case ANALYSE:
|
||||
result = analyzeService.analyze(analyzeRequest);
|
||||
log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
|
||||
case SURROUNDING_TEXT:
|
||||
var manualRedactions = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
|
||||
result = AnalyzeResult.builder()
|
||||
.dossierId(analyzeRequest.getDossierId())
|
||||
.fileId(analyzeRequest.getFileId())
|
||||
.manualRedactions(manualRedactions)
|
||||
.build();
|
||||
result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
|
||||
log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
+334
-351
@@ -8,11 +8,9 @@ import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@@ -74,9 +72,7 @@ public class Section {
|
||||
|
||||
public void addAiEntities(String type, String asType) {
|
||||
|
||||
Set<Entity> entitiesOfType = nerEntities.stream()
|
||||
.filter(nerEntity -> nerEntity.getType().equals(type))
|
||||
.collect(Collectors.toSet());
|
||||
Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
|
||||
Set<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toSet());
|
||||
Set<Entity> found = EntitySearchUtils.find(searchText, values, asType, headline, sectionNumber, false, false, Engine.NER, true);
|
||||
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
@@ -84,13 +80,12 @@ public class Section {
|
||||
Set<Entity> finalResult = new HashSet<>();
|
||||
|
||||
// Only keep Entities with correct offsets from AI Service.
|
||||
Iterator<Entity> itty = found.iterator();
|
||||
while (itty.hasNext()) {
|
||||
Entity current = itty.next();
|
||||
for (Entity current : found) {
|
||||
boolean foundSameOffsets = false;
|
||||
for (Entity entity : entitiesOfType) {
|
||||
if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) {
|
||||
foundSameOffsets = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (foundSameOffsets) {
|
||||
@@ -98,20 +93,18 @@ public class Section {
|
||||
}
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, finalResult, dictionary);
|
||||
var nonOverlapping = EntitySearchUtils.findNonOverlappingMatchEntities(entities, finalResult);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, nonOverlapping, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
nerEntities.removeAll(entitiesOfType);
|
||||
}
|
||||
|
||||
|
||||
public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType,
|
||||
int minPartMatches, boolean allowDuplicateTypes) {
|
||||
public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType, int minPartMatches, boolean allowDuplicateTypes) {
|
||||
|
||||
Set<String> combineSet = Set.of(combineTypes.split(","));
|
||||
|
||||
List<Entity> sorted = nerEntities.stream()
|
||||
.sorted(Comparator.comparing(Entity::getStart))
|
||||
.collect(Collectors.toList());
|
||||
List<Entity> sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
|
||||
Set<Entity> found = new HashSet<>();
|
||||
int start = -1;
|
||||
int lastEnd = -1;
|
||||
@@ -162,87 +155,53 @@ public class Section {
|
||||
}
|
||||
|
||||
if (!found.isEmpty()) {
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
var nonOverlapping = EntitySearchUtils.findNonOverlappingMatchEntities(entities, found);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, nonOverlapping, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.filter(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.filter(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.filter(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.filter(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
.anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
return fileAttributes != null && fileAttributes.stream()
|
||||
.filter(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()))
|
||||
.findFirst()
|
||||
.isPresent();
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
|
||||
.toString()
|
||||
.equals(value);
|
||||
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
|
||||
}
|
||||
|
||||
|
||||
@@ -282,21 +241,26 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group) {
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
|
||||
|
||||
expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.REGEX) String valuePattern) {
|
||||
|
||||
Pattern compiledValuePattern = null;
|
||||
@@ -339,94 +303,169 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactImage(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
images.forEach(image -> {
|
||||
if (image.getType().equals(type)) {
|
||||
image.setRedaction(true);
|
||||
image.setMatchedRule(ruleNumber);
|
||||
image.setRedactionReason(reason);
|
||||
image.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
redactImage(type, ruleNumber, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactImage(type, ruleNumber, reason, null, false);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
redact(type, ruleNumber, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotImage(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
images.forEach(image -> {
|
||||
if (image.getType().equals(type)) {
|
||||
image.setRedaction(false);
|
||||
image.setMatchedRule(ruleNumber);
|
||||
image.setRedactionReason(reason);
|
||||
}
|
||||
});
|
||||
redact(type, ruleNumber, reason, null, false);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(false);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void ignore(String type) {
|
||||
|
||||
entities.removeIf(entity -> entity.getType().equals(type));
|
||||
redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
Set<Entity> references = entities.stream()
|
||||
.filter(entity -> entity.getType().equals(referenceType))
|
||||
.collect(Collectors.toSet());
|
||||
Set<Entity> references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet());
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType().equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(false);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
@@ -437,10 +476,38 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.STRING) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
|
||||
public void ignore(String type) {
|
||||
|
||||
entities.removeIf(entity -> entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void expandToHintAnnotationByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
@@ -468,10 +535,8 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
@@ -488,26 +553,7 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix,
|
||||
@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
@@ -515,43 +561,7 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void recommendLineAfter(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
@@ -567,8 +577,7 @@ public class Section {
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>())
|
||||
.add(cleanValue);
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(cleanValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -576,163 +585,29 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(searchText, start, stop);
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
|
||||
@Argument(ArgumentType.STRING) String stop,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, stop);
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String[] lines = value.split("\n");
|
||||
|
||||
for (String line : lines) {
|
||||
|
||||
if (line.trim().length() <= 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.TYPE) String type) {
|
||||
public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactCell(@Argument(ArgumentType.STRING) String cellHeader,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
|
||||
boolean redaction) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
|
||||
@Argument(ArgumentType.STRING) String reason) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
|
||||
int ruleNumber, String reason, String legalBasis, Engine engine) {
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine) {
|
||||
|
||||
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseInsensitive ? value.toLowerCase() : value;
|
||||
@@ -752,8 +627,35 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
|
||||
boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType().equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(redaction);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private void redactImage(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
images.forEach(image -> {
|
||||
if (image.getType().equals(type)) {
|
||||
image.setRedaction(redaction);
|
||||
image.setMatchedRule(ruleNumber);
|
||||
image.setRedactionReason(reason);
|
||||
image.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
|
||||
String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
@@ -790,11 +692,92 @@ public class Section {
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group().trim();
|
||||
if (match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>())
|
||||
.add(match);
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()).add(match);
|
||||
String lastname = match.split(" ")[0];
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>())
|
||||
.add(lastname);
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>()).add(lastname);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(searchText, start, stop);
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, stop);
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String[] lines = value.split("\n");
|
||||
|
||||
for (String line : lines) {
|
||||
|
||||
if (line.trim().length() <= 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+103
-79
@@ -1,34 +1,55 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -46,12 +67,11 @@ public class AnalyzeService {
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
private final SectionTextBuilderService sectionTextBuilderService;
|
||||
private final SectionGridCreatorService sectionGridCreatorService;
|
||||
private final NerAnalyserService nerAnalyserService;
|
||||
private final ImageService imageService;
|
||||
private final ImportedRedactionService importedRedactionService;
|
||||
|
||||
|
||||
public void analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
|
||||
public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
@@ -59,9 +79,7 @@ public class AnalyzeService {
|
||||
Document classifiedDoc;
|
||||
|
||||
try {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
|
||||
.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
|
||||
|
||||
Map<Integer, List<PdfImage>> pdfImages = null;
|
||||
if (redactionServiceSettings.isEnableImageClassification()) {
|
||||
@@ -79,51 +97,24 @@ public class AnalyzeService {
|
||||
Text text = new Text(pageCount, sectionTexts);
|
||||
|
||||
// enhance section grid with headline data
|
||||
sectionTexts.forEach(sectionText -> classifiedDoc
|
||||
.getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
|
||||
sectionText.getHeadline(),
|
||||
sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
|
||||
sectionText.getSectionAreas())));
|
||||
sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid()
|
||||
.getSections()
|
||||
.add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
|
||||
.stream()
|
||||
.map(SectionArea::getPage)
|
||||
.collect(Collectors.toSet()), sectionText.getSectionAreas())));
|
||||
|
||||
log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
|
||||
.getSectionGrid());
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
|
||||
|
||||
log.info("Document structure analysis successful, took: {}", System.currentTimeMillis() - startTime);
|
||||
}
|
||||
|
||||
|
||||
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
|
||||
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
|
||||
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
|
||||
.getDossierId());
|
||||
|
||||
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
|
||||
|
||||
dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
|
||||
.getDossierTemplateId());
|
||||
|
||||
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
|
||||
redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
|
||||
rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
|
||||
importedRedactionService.processImportedRedactions(redactionLog, analyzeRequest.getImportedAnnotations(), analyzeRequest.getDossierTemplateId());
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
|
||||
return AnalyzeResult.builder()
|
||||
.dossierId(analyzeRequest.getDossierId())
|
||||
.fileId(analyzeRequest.getFileId())
|
||||
.duration(System.currentTimeMillis() - startTime)
|
||||
.numberOfPages(text.getNumberOfPages())
|
||||
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -140,8 +131,7 @@ public class AnalyzeService {
|
||||
return analyze(analyzeRequest);
|
||||
}
|
||||
|
||||
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
|
||||
.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
|
||||
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
|
||||
|
||||
Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse()
|
||||
.isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
|
||||
@@ -150,10 +140,11 @@ public class AnalyzeService {
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
|
||||
}
|
||||
|
||||
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
|
||||
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
NerEntities nerEntities;
|
||||
if (redactionServiceSettings.isNerServiceEnabled()) {
|
||||
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
} else {
|
||||
nerEntities = NerEntities.builder().build();
|
||||
}
|
||||
|
||||
List<SectionText> reanalysisSections = text.getSectionTexts()
|
||||
@@ -163,20 +154,54 @@ public class AnalyzeService {
|
||||
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
|
||||
.getDossierId());
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
|
||||
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
|
||||
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
|
||||
.getDossierTemplateId());
|
||||
var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
|
||||
|
||||
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false);
|
||||
|
||||
redactionLog.getRedactionLogEntry()
|
||||
.removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType()
|
||||
.equals(IMPORTED_REDACTION_TYPE));
|
||||
redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
|
||||
|
||||
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
|
||||
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
|
||||
importedRedactionService.processImportedRedactions(redactionLog, analyzeRequest.getImportedAnnotations(), analyzeRequest.getDossierTemplateId());
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
|
||||
}
|
||||
|
||||
|
||||
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
|
||||
NerEntities nerEntities;
|
||||
if (redactionServiceSettings.isNerServiceEnabled()) {
|
||||
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
} else {
|
||||
nerEntities = NerEntities.builder().build();
|
||||
}
|
||||
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
|
||||
PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
|
||||
|
||||
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, legalBasis, dictionary.getVersion()
|
||||
.getDossierTemplateVersion(), dictionary.getVersion()
|
||||
.getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
|
||||
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog.getRedactionLogEntry(), true);
|
||||
redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
|
||||
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
|
||||
}
|
||||
|
||||
|
||||
private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog,
|
||||
Text text, AnalyzeRequest analyzeRequest) {
|
||||
|
||||
@@ -217,10 +242,8 @@ public class AnalyzeService {
|
||||
|
||||
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
|
||||
|
||||
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
|
||||
redactionLog, analyzeRequest.getAnalysisNumber());
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
|
||||
.getRedactionLog());
|
||||
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog, analyzeRequest.getAnalysisNumber());
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
|
||||
@@ -249,13 +272,14 @@ public class AnalyzeService {
|
||||
}
|
||||
|
||||
return Stream.concat(manualRedactions.getLegalBasisChanges()
|
||||
.stream()
|
||||
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
|
||||
.stream()
|
||||
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
|
||||
.stream()
|
||||
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
|
||||
.collect(Collectors.toSet());
|
||||
.stream()
|
||||
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
|
||||
.stream()
|
||||
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
|
||||
.stream()
|
||||
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions()
|
||||
.stream()
|
||||
.map(ManualForceRedaction::getAnnotationId)))).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
|
||||
+32
-22
@@ -5,7 +5,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import feign.FeignException;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -36,13 +35,13 @@ public class DictionaryService {
|
||||
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
|
||||
var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
|
||||
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
|
||||
updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, null);
|
||||
updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
|
||||
}
|
||||
|
||||
long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
|
||||
var dossierDictionary = dictionariesByDossier.get(dossierId);
|
||||
if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
|
||||
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId);
|
||||
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
|
||||
}
|
||||
|
||||
return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
|
||||
@@ -78,7 +77,7 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) {
|
||||
private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
|
||||
|
||||
try {
|
||||
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
|
||||
@@ -88,8 +87,26 @@ public class DictionaryService {
|
||||
|
||||
List<DictionaryModel> dictionary = typeResponse
|
||||
.stream()
|
||||
.map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
|
||||
.isHint(), t.isRecommendation(), convertEntries(t.getId()), new HashSet<>(), dossierId != null))
|
||||
.map(t -> {
|
||||
|
||||
Optional<DictionaryModel> oldModel;
|
||||
if (dossierId == null) {
|
||||
var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
|
||||
oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
|
||||
} else {
|
||||
var representation = dictionariesByDossier.get(dossierId);
|
||||
oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
|
||||
}
|
||||
|
||||
Set<DictionaryEntry> entries = new HashSet<>();
|
||||
// add old entries from existing DictionaryModel
|
||||
oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries()));
|
||||
// Add Increments
|
||||
entries.addAll(convertEntries(t.getId(), currentVersion));
|
||||
|
||||
return new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
|
||||
.isHint(), t.isRecommendation(), entries, new HashSet<>(), dossierId != null);
|
||||
})
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
@@ -118,23 +135,9 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public void updateExternalDictionary(Dictionary dictionary, String dossierTemplateId, String dossierId) {
|
||||
private Set<DictionaryEntry> convertEntries(String typeId, Long fromVersion) {
|
||||
|
||||
dictionary.getDictionaryModels().forEach(dm -> {
|
||||
if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
|
||||
dictionaryClient.addEntries(IdBuilder.getTypeId(dm.getType(), dossierTemplateId, dm.isDossierDictionary() ? dossierId : null), new ArrayList<>(dm.getLocalEntries()), false, true);
|
||||
long externalVersion = dictionaryClient.getVersion(dossierTemplateId);
|
||||
if (externalVersion == dictionary.getVersion().getDossierTemplateVersion() + 1) {
|
||||
dictionary.getVersion().setDossierTemplateVersion(externalVersion);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private Set<DictionaryEntry> convertEntries(String typeId) {
|
||||
|
||||
var type = dictionaryClient.getDictionaryForType(typeId);
|
||||
var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);
|
||||
|
||||
Set<DictionaryEntry> entries = new HashSet<>(type
|
||||
.getEntries());
|
||||
@@ -233,4 +236,11 @@ public class DictionaryService {
|
||||
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
|
||||
}
|
||||
|
||||
private Long getVersion(DictionaryRepresentation dictionaryRepresentation) {
|
||||
if (dictionaryRepresentation == null) {
|
||||
return null;
|
||||
} else {
|
||||
return dictionaryRepresentation.getDictionaryVersion();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+35
-21
@@ -1,5 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
@@ -8,20 +21,21 @@ import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -46,7 +60,7 @@ public class EntityRedactionService {
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
|
||||
Map<Integer, List<Entity>> entitiesPerPage = convertToEnititesPerPage(entities);
|
||||
Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(entities);
|
||||
return new PageEntities(entitiesPerPage, imagesPerPage);
|
||||
}
|
||||
|
||||
@@ -145,7 +159,7 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
|
||||
private Map<Integer, List<Entity>> convertToEnititesPerPage(Set<Entity> entities) {
|
||||
private Map<Integer, List<Entity>> convertToEntitiesPerPage(Set<Entity> entities) {
|
||||
|
||||
Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
|
||||
for (Entity entity : entities) {
|
||||
@@ -209,7 +223,7 @@ public class EntityRedactionService {
|
||||
|
||||
private Entities findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
Dictionary dictionary, boolean local, NerEntities nerEntities,
|
||||
List<Integer> cellstarts) {
|
||||
List<Integer> cellStarts) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
String searchableString = searchableText.toString();
|
||||
@@ -230,32 +244,32 @@ public class EntityRedactionService {
|
||||
|
||||
Set<Entity> nerFound = new HashSet<>();
|
||||
if (!local) {
|
||||
nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellstarts, headline));
|
||||
nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
|
||||
}
|
||||
|
||||
return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities,
|
||||
List<Integer> cellstarts, String headline) {
|
||||
private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts,
|
||||
String headline) {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
|
||||
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities.getResult()
|
||||
.containsKey(sectionNumber)) {
|
||||
if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getResult().containsKey(sectionNumber)) {
|
||||
nerEntities.getResult().get(sectionNumber).forEach(res -> {
|
||||
if (cellstarts == null || cellstarts.isEmpty()) {
|
||||
entities.add(new Entity(new String(Base64.decodeBase64(res.getValue().getBytes())), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
|
||||
if (cellStarts == null || cellStarts.isEmpty()) {
|
||||
entities.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
|
||||
} else {
|
||||
boolean intersectsCellStart = false;
|
||||
for (Integer cellStart : cellstarts) {
|
||||
for (Integer cellStart : cellStarts) {
|
||||
if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
|
||||
intersectsCellStart = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!intersectsCellStart) {
|
||||
entities.add(new Entity(new String(Base64.decodeBase64(res.getValue().getBytes())), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
|
||||
entities.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
+63
-59
@@ -2,16 +2,14 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotationStatus;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -19,80 +17,86 @@ import lombok.RequiredArgsConstructor;
|
||||
@RequiredArgsConstructor
|
||||
public class ImportedRedactionService {
|
||||
|
||||
public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
public void processImportedRedactions(RedactionLog redactionLog, List<ImportedAnnotation> importedAnnotations, String dossierTemplateId) {
|
||||
|
||||
Iterator<RedactionLogEntry> itty = redactionLog.getRedactionLogEntry().iterator();
|
||||
while (itty.hasNext()) {
|
||||
RedactionLogEntry redactionLogEntry = itty.next();
|
||||
boolean mustBeRemoved = false;
|
||||
public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId, String dossierId, String fileId,
|
||||
List<RedactionLogEntry> redactionLogEntries, boolean addImportedRedactions) {
|
||||
|
||||
if (redactionLogEntry.getType().equals("imported_redaction")) {
|
||||
itty.remove();
|
||||
continue;
|
||||
var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
|
||||
if (importedRedactions == null) {
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
redactionLogEntries.removeIf(redactionLogEntry -> hasIntersections(redactionLogEntry, importedRedactions));
|
||||
|
||||
if (addImportedRedactions) {
|
||||
return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
|
||||
}
|
||||
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
|
||||
private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
|
||||
List<RedactionLogEntry> redactionLogEntries,
|
||||
ImportedRedactions importedRedactions) {
|
||||
|
||||
for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
|
||||
for (ImportedRedaction importedRedaction : importedRedactionsValues) {
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(importedRedaction.getId())
|
||||
.type(IMPORTED_REDACTION_TYPE)
|
||||
.imported(true)
|
||||
.redacted(true)
|
||||
.positions(importedRedaction.getPositions())
|
||||
.color(getColor("imported_redaction", dossierTemplateId))
|
||||
.build();
|
||||
|
||||
redactionLogEntries.add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
loop:
|
||||
for (ImportedAnnotation importedAnnotation : importedAnnotations) {
|
||||
for (com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition : importedAnnotation.getPositions()) {
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
if (intersects(importedPosition, rectangle)) {
|
||||
mustBeRemoved = true;
|
||||
break loop;
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
|
||||
private boolean hasIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
|
||||
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
|
||||
var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
|
||||
for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
|
||||
for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
|
||||
if (intersects(importedRedactionPosition, rectangle)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (mustBeRemoved) {
|
||||
itty.remove();
|
||||
}
|
||||
}
|
||||
|
||||
for (ImportedAnnotation importedAnnotation : importedAnnotations) {
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(importedAnnotation.getAnnotationId())
|
||||
.type("imported_redaction")
|
||||
.redacted(importedAnnotation.getStatus().equals(ImportedAnnotationStatus.DECLINED) ? false : true)
|
||||
.positions(importedAnnotation.getPositions()
|
||||
.stream()
|
||||
.map(this::convert)
|
||||
.collect(Collectors.toList()))
|
||||
.color(getColor("imported_redaction", dossierTemplateId, importedAnnotation.getStatus().equals(ImportedAnnotationStatus.DECLINED) ? false : true))
|
||||
.build();
|
||||
|
||||
redactionLog.getRedactionLogEntry().add(redactionLogEntry);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public Rectangle convert(
|
||||
com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition) {
|
||||
private boolean intersects(Rectangle importedPosition, Rectangle redactionLogPosition) {
|
||||
|
||||
return new Rectangle(new Point(importedPosition.getTopLeftX(), importedPosition.getTopLeftY()), importedPosition.getWidth(), importedPosition.getHeight(), importedPosition.getPage());
|
||||
return redactionLogPosition.getTopLeft()
|
||||
.getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeft()
|
||||
.getX() && redactionLogPosition.getTopLeft()
|
||||
.getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeft()
|
||||
.getY() && redactionLogPosition.getTopLeft().getX() < importedPosition.getTopLeft()
|
||||
.getX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft()
|
||||
.getY() < importedPosition.getTopLeft().getY() + importedPosition.getHeight();
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(
|
||||
com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition,
|
||||
Rectangle redactionLogPosition) {
|
||||
private float[] getColor(String type, String dossierTemplateId) {
|
||||
|
||||
return importedPosition.getPage() == redactionLogPosition.getPage() && redactionLogPosition.getTopLeft()
|
||||
.getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeftX() && redactionLogPosition.getTopLeft()
|
||||
.getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeftY() && redactionLogPosition.getTopLeft()
|
||||
.getX() < importedPosition.getTopLeftX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft()
|
||||
.getY() < importedPosition.getTopLeftY() + importedPosition.getHeight();
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
|
||||
|
||||
if (!isRedaction) {
|
||||
return dictionaryService.getNotRedactedColor(dossierTemplateId);
|
||||
}
|
||||
return dictionaryService.getColor(type, dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+22
-16
@@ -1,9 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
@@ -13,14 +21,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -31,8 +34,9 @@ public class ManualRedactionSurroundingTextService {
|
||||
private final SurroundingWordsService surroundingWordsService;
|
||||
|
||||
|
||||
public ManualRedactions addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
|
||||
public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
Text text = redactionStorageService.getText(dossierId, fileId);
|
||||
List<ManualRedactionEntry> processedAddRedactions = new ArrayList<>();
|
||||
List<ManualResizeRedaction> processedResizeRedactions = new ArrayList<>();
|
||||
@@ -49,8 +53,7 @@ public class ManualRedactionSurroundingTextService {
|
||||
while (addItty.hasNext()) {
|
||||
var manualAddRedaction = addItty.next();
|
||||
if (sectionContainsEntry(sectionArea, manualAddRedaction.getPositions())) {
|
||||
var surroundingText = findSurroundingText(sectionText, manualAddRedaction.getValue(), manualAddRedaction
|
||||
.getPositions());
|
||||
var surroundingText = findSurroundingText(sectionText, manualAddRedaction.getValue(), manualAddRedaction.getPositions());
|
||||
manualAddRedaction.setTextBefore(surroundingText.getLeft());
|
||||
manualAddRedaction.setTextAfter(surroundingText.getRight());
|
||||
processedAddRedactions.add(manualAddRedaction);
|
||||
@@ -62,8 +65,7 @@ public class ManualRedactionSurroundingTextService {
|
||||
while (resizeItty.hasNext()) {
|
||||
var manualResizeRedaction = resizeItty.next();
|
||||
if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
|
||||
var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction
|
||||
.getPositions());
|
||||
var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
|
||||
manualResizeRedaction.setTextBefore(surroundingText.getLeft());
|
||||
manualResizeRedaction.setTextAfter(surroundingText.getRight());
|
||||
processedResizeRedactions.add(manualResizeRedaction);
|
||||
@@ -75,15 +77,20 @@ public class ManualRedactionSurroundingTextService {
|
||||
|
||||
manualRedactions.getEntriesToAdd().addAll(processedAddRedactions);
|
||||
manualRedactions.getResizeRedactions().addAll(processedResizeRedactions);
|
||||
return manualRedactions;
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.manualRedactions(manualRedactions)
|
||||
.duration(System.currentTimeMillis() - startTime)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private Pair<String, String> findSurroundingText(SectionText sectionText, String value,
|
||||
List<Rectangle> toFindPositions) {
|
||||
|
||||
Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText
|
||||
.getSectionNumber(), false, false, Engine.DICTIONARY, false);
|
||||
Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, false);
|
||||
Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);
|
||||
|
||||
Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);
|
||||
@@ -94,8 +101,7 @@ public class ManualRedactionSurroundingTextService {
|
||||
}
|
||||
|
||||
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
|
||||
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText
|
||||
.getCellStarts());
|
||||
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
|
||||
} else {
|
||||
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
|
||||
}
|
||||
|
||||
-53
@@ -1,53 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class NerAnalyserService {
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
private final EntityRecognitionClient entityRecognitionClient;
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
public void computeNerEntities(String dossierId, String fileId) {
|
||||
|
||||
if (redactionServiceSettings.isEnableEntityRecognition()) {
|
||||
var text = redactionStorageService.getText(dossierId, fileId);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
if (text != null) {
|
||||
var nerRequest = EntityRecognitionRequest.builder()
|
||||
.data(text.getSectionTexts()
|
||||
.stream()
|
||||
.map(sectionText -> new EntityRecognitionSection(sectionText.getSectionNumber(), new String(Base64
|
||||
.encodeBase64(sectionText
|
||||
.getText().getBytes()))))
|
||||
.collect(Collectors.toList()))
|
||||
.build();
|
||||
|
||||
var nerResponse = entityRecognitionClient.findAuthors(nerRequest);
|
||||
|
||||
log.info("Computing NER entities took: {} ms for dossierId {} and fileId {}", System.currentTimeMillis() - start, dossierId, fileId);
|
||||
|
||||
redactionStorageService.storeObject(dossierId, fileId, FileType.NER_ENTITIES, nerResponse);
|
||||
} else {
|
||||
log.warn("Warning, text for file: {} in dossier: {} is null", fileId, dossierId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+12
-6
@@ -1,5 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
@@ -10,13 +20,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -85,7 +91,7 @@ public class RedactionLogCreatorService {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
|
||||
// Duplicates can exist due table extraction colums over multiple rows.
|
||||
// Duplicates can exist due table extraction columns over multiple rows.
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
|
||||
entityLoop:
|
||||
|
||||
+6
-3
@@ -306,7 +306,6 @@ public class RedactionLogMergeService {
|
||||
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) &&
|
||||
manualRedactionEntry.getProcessedDate() != null &&
|
||||
(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
|
||||
}
|
||||
|
||||
@@ -316,6 +315,11 @@ public class RedactionLogMergeService {
|
||||
|
||||
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
|
||||
|
||||
var change = ManualChange.from(manualRedactionEntry)
|
||||
.withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
|
||||
List<ManualChange> changeList = new ArrayList<>();
|
||||
changeList.add(change);
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
|
||||
@@ -330,8 +334,7 @@ public class RedactionLogMergeService {
|
||||
.isHint(false)
|
||||
.sectionNumber(-1)
|
||||
.rectangle(manualRedactionEntry.isRectangle())
|
||||
.manualChanges(List.of(ManualChange.from(manualRedactionEntry)
|
||||
.withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)))
|
||||
.manualChanges(changeList)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
+6
-1
@@ -128,7 +128,12 @@ public class EntitySearchUtils {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
|
||||
wordsToRemove.add(inner);
|
||||
// FIXME this is workaround for RED-3327 and should be removed in the future.
|
||||
if(word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
|
||||
wordsToRemove.add(word);
|
||||
} else {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
-2
@@ -1,8 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.settings;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties("redaction-service")
|
||||
public class RedactionServiceSettings {
|
||||
@@ -17,6 +18,6 @@ public class RedactionServiceSettings {
|
||||
|
||||
private int analysisVersion = 1;
|
||||
|
||||
private boolean enableEntityRecognition = true;
|
||||
private boolean nerServiceEnabled = true;
|
||||
|
||||
}
|
||||
|
||||
+21
-5
@@ -8,6 +8,7 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
@@ -44,6 +45,24 @@ public class RedactionStorageService {
|
||||
}
|
||||
|
||||
|
||||
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Imported redactions not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not imported redactions", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public RedactionLog getRedactionLog(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
@@ -86,14 +105,13 @@ public class RedactionStorageService {
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("NER Entities not available.");
|
||||
return null;
|
||||
throw new NotFoundException("NER Entities are not available.");
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(inputStreamResource.getInputStream(), NerEntities.class);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert NerEntities", e);
|
||||
throw new RuntimeException("Could not convert NER Entities", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,8 +145,6 @@ public class RedactionStorageService {
|
||||
return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+411
-417
@@ -1,31 +1,33 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
@@ -50,42 +52,15 @@ import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@@ -215,6 +190,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
loadDictionaryForTest();
|
||||
loadTypeForTest();
|
||||
loadNerForTest();
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getTypeResponse());
|
||||
|
||||
@@ -230,319 +206,40 @@ public class RedactionIntegrationTest {
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.build()));
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SPONSOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
|
||||
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PII, false));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
|
||||
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PURITY, false));
|
||||
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMAGE, false));
|
||||
when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(OCR, false));
|
||||
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(LOGO, false));
|
||||
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false));
|
||||
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
|
||||
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
|
||||
mockDictionaryCalls(null);
|
||||
mockDictionaryCalls(0L);
|
||||
|
||||
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
|
||||
private void mockDictionaryCalls(Long version){
|
||||
|
||||
private void loadDictionaryForTest() {
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(SPONSOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
|
||||
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
|
||||
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
|
||||
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMAGE, false));
|
||||
when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(OCR, false));
|
||||
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(LOGO, false));
|
||||
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SIGNATURE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FORMULA, false));
|
||||
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
|
||||
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/hint_only.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/must_redact.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/published_information.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/test_method.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(RECOMMENDATION_AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/recommendation_CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(RECOMMENDATION_ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/recommendation_CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FALSE_POSITIVE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/false_positive.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
|
||||
}
|
||||
|
||||
|
||||
private String cleanDictionaryEntry(String entry) {
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
|
||||
}
|
||||
|
||||
|
||||
private void loadTypeForTest() {
|
||||
|
||||
typeColorMap.put(VERTEBRATE, "#ff85f7");
|
||||
typeColorMap.put(ADDRESS, "#ffe187");
|
||||
typeColorMap.put(AUTHOR, "#ffe187");
|
||||
typeColorMap.put(SPONSOR, "#85ebff");
|
||||
typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
|
||||
typeColorMap.put(REDACTION_INDICATOR, "#caff85");
|
||||
typeColorMap.put(HINT_ONLY, "#abc0c4");
|
||||
typeColorMap.put(MUST_REDACT, "#fab4c0");
|
||||
typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
|
||||
typeColorMap.put(TEST_METHOD, "#91fae8");
|
||||
typeColorMap.put(PII, "#66ccff");
|
||||
typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
|
||||
typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
|
||||
typeColorMap.put(FALSE_POSITIVE, "#ffffff");
|
||||
typeColorMap.put(PURITY, "#ffe187");
|
||||
typeColorMap.put(IMAGE, "#fcc5fb");
|
||||
typeColorMap.put(OCR, "#fcc5fb");
|
||||
typeColorMap.put(LOGO, "#ffe187");
|
||||
typeColorMap.put(FORMULA, "#ffe187");
|
||||
typeColorMap.put(SIGNATURE, "#ffe187");
|
||||
typeColorMap.put(IMPORTED_REDACTION, "#32a852");
|
||||
|
||||
hintTypeMap.put(VERTEBRATE, true);
|
||||
hintTypeMap.put(ADDRESS, false);
|
||||
hintTypeMap.put(AUTHOR, false);
|
||||
hintTypeMap.put(SPONSOR, false);
|
||||
hintTypeMap.put(NO_REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(HINT_ONLY, true);
|
||||
hintTypeMap.put(MUST_REDACT, true);
|
||||
hintTypeMap.put(PUBLISHED_INFORMATION, true);
|
||||
hintTypeMap.put(TEST_METHOD, true);
|
||||
hintTypeMap.put(PII, false);
|
||||
hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
hintTypeMap.put(FALSE_POSITIVE, true);
|
||||
hintTypeMap.put(PURITY, false);
|
||||
hintTypeMap.put(IMAGE, true);
|
||||
hintTypeMap.put(OCR, true);
|
||||
hintTypeMap.put(FORMULA, false);
|
||||
hintTypeMap.put(LOGO, false);
|
||||
hintTypeMap.put(SIGNATURE, false);
|
||||
hintTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
hintTypeMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
caseInSensitiveMap.put(VERTEBRATE, true);
|
||||
caseInSensitiveMap.put(ADDRESS, false);
|
||||
caseInSensitiveMap.put(AUTHOR, false);
|
||||
caseInSensitiveMap.put(SPONSOR, false);
|
||||
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(HINT_ONLY, true);
|
||||
caseInSensitiveMap.put(MUST_REDACT, true);
|
||||
caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
|
||||
caseInSensitiveMap.put(TEST_METHOD, false);
|
||||
caseInSensitiveMap.put(PII, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
caseInSensitiveMap.put(FALSE_POSITIVE, false);
|
||||
caseInSensitiveMap.put(PURITY, false);
|
||||
caseInSensitiveMap.put(IMAGE, true);
|
||||
caseInSensitiveMap.put(OCR, true);
|
||||
caseInSensitiveMap.put(SIGNATURE, true);
|
||||
caseInSensitiveMap.put(LOGO, true);
|
||||
caseInSensitiveMap.put(FORMULA, true);
|
||||
caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
|
||||
caseInSensitiveMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
recommendationTypeMap.put(VERTEBRATE, false);
|
||||
recommendationTypeMap.put(ADDRESS, false);
|
||||
recommendationTypeMap.put(AUTHOR, false);
|
||||
recommendationTypeMap.put(SPONSOR, false);
|
||||
recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(HINT_ONLY, false);
|
||||
recommendationTypeMap.put(MUST_REDACT, false);
|
||||
recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
|
||||
recommendationTypeMap.put(TEST_METHOD, false);
|
||||
recommendationTypeMap.put(PII, false);
|
||||
recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
|
||||
recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
|
||||
recommendationTypeMap.put(FALSE_POSITIVE, false);
|
||||
recommendationTypeMap.put(PURITY, false);
|
||||
recommendationTypeMap.put(IMAGE, false);
|
||||
recommendationTypeMap.put(OCR, false);
|
||||
recommendationTypeMap.put(FORMULA, false);
|
||||
recommendationTypeMap.put(SIGNATURE, false);
|
||||
recommendationTypeMap.put(LOGO, false);
|
||||
recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
recommendationTypeMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
rankTypeMap.put(FALSE_POSITIVE, 160);
|
||||
rankTypeMap.put(PURITY, 155);
|
||||
rankTypeMap.put(PII, 150);
|
||||
rankTypeMap.put(ADDRESS, 140);
|
||||
rankTypeMap.put(AUTHOR, 130);
|
||||
rankTypeMap.put(SPONSOR, 120);
|
||||
rankTypeMap.put(VERTEBRATE, 110);
|
||||
rankTypeMap.put(MUST_REDACT, 100);
|
||||
rankTypeMap.put(REDACTION_INDICATOR, 90);
|
||||
rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
|
||||
rankTypeMap.put(PUBLISHED_INFORMATION, 70);
|
||||
rankTypeMap.put(TEST_METHOD, 60);
|
||||
rankTypeMap.put(HINT_ONLY, 50);
|
||||
rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
|
||||
rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
|
||||
rankTypeMap.put(IMAGE, 30);
|
||||
rankTypeMap.put(OCR, 29);
|
||||
rankTypeMap.put(LOGO, 28);
|
||||
rankTypeMap.put(SIGNATURE, 27);
|
||||
rankTypeMap.put(FORMULA, 26);
|
||||
rankTypeMap.put(DOSSIER_REDACTIONS, 200);
|
||||
rankTypeMap.put(IMPORTED_REDACTION, 200);
|
||||
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
colors.setRequestAdd("#04b093");
|
||||
colors.setRequestRemove("#04b093");
|
||||
}
|
||||
|
||||
|
||||
private List<Type> getTypeResponse() {
|
||||
|
||||
return typeColorMap.entrySet()
|
||||
.stream()
|
||||
.map(typeColor -> Type.builder()
|
||||
.id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.type(typeColor.getKey())
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColor.getValue())
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
|
||||
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
|
||||
.rank(rankTypeMap.get(typeColor.getKey()))
|
||||
.build())
|
||||
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
|
||||
|
||||
return Type.builder()
|
||||
.id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColorMap.get(type))
|
||||
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
|
||||
.isHint(hintTypeMap.get(type))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(type))
|
||||
.isRecommendation(recommendationTypeMap.get(type))
|
||||
.rank(rankTypeMap.get(type))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
|
||||
|
||||
List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
|
||||
entries.forEach(entry -> {
|
||||
dictionaryEntries.add(DictionaryEntry.builder()
|
||||
.value(entry)
|
||||
.version(reanlysisVersions.getOrDefault(entry, 0L))
|
||||
.deleted(deleted.contains(entry))
|
||||
.build());
|
||||
});
|
||||
return dictionaryEntries;
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void test270Rotated() {
|
||||
|
||||
@@ -584,9 +281,7 @@ public class RedactionIntegrationTest {
|
||||
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
|
||||
});
|
||||
|
||||
duplicates.entrySet().forEach(entry -> {
|
||||
assertThat(entry.getValue().size()).isEqualTo(1);
|
||||
});
|
||||
duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
|
||||
|
||||
dictionary.get(AUTHOR).add("Drinking water");
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
|
||||
@@ -804,9 +499,9 @@ public class RedactionIntegrationTest {
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, null)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
|
||||
@@ -842,7 +537,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
@@ -867,17 +562,18 @@ public class RedactionIntegrationTest {
|
||||
dictionary.get(AUTHOR).add("report");
|
||||
reanlysisVersions.put("report", 2L);
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
mockDictionaryCalls(0L);
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
dictionary.get(AUTHOR).add("assessment report");
|
||||
reanlysisVersions.put("assessment report", 3L);
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
mockDictionaryCalls(2L);
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
mockDictionaryCalls(3L);
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
@@ -905,9 +601,13 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void redactionTest() throws IOException {
|
||||
|
||||
String fileName = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||
String fileName = "files/new/crafted document.pdf";
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(fileName);
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
@@ -957,6 +657,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
dictionary.get(AUTHOR).add("properties");
|
||||
reanlysisVersions.put("properties", 1L);
|
||||
mockDictionaryCalls(0L);
|
||||
|
||||
dictionary.get(AUTHOR).add("physical");
|
||||
reanlysisVersions.put("physical", 2L);
|
||||
@@ -971,9 +672,9 @@ public class RedactionIntegrationTest {
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, null)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
|
||||
@@ -1006,10 +707,11 @@ public class RedactionIntegrationTest {
|
||||
|
||||
deleted.remove("mouse");
|
||||
reanlysisVersions.put("mouse", 4L);
|
||||
mockDictionaryCalls(3L);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
@@ -1047,8 +749,6 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void testManualRedaction() throws IOException {
|
||||
|
||||
// 675eba69b0c2917de55462c817adaa05
|
||||
|
||||
System.out.println("testManualRedaction");
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
|
||||
@@ -1099,8 +799,6 @@ public class RedactionIntegrationTest {
|
||||
.page(1)
|
||||
.build()));
|
||||
|
||||
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
@@ -1246,34 +944,6 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(String file) {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
||||
|
||||
return prepareStorage(pdfFileResource.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(InputStream stream) {
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
|
||||
return request;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void sponsorCompanyTest() throws IOException {
|
||||
|
||||
@@ -1527,7 +1197,8 @@ public class RedactionIntegrationTest {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions);
|
||||
var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions)
|
||||
.getManualRedactions();
|
||||
surroundingTextResult.getEntriesToAdd().forEach(addEntry -> {
|
||||
assertThat(addEntry.getTextAfter()).isNotEmpty();
|
||||
});
|
||||
@@ -1535,6 +1206,117 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
private void loadDictionaryForTest() {
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/hint_only.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/must_redact.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/published_information.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/test_method.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(RECOMMENDATION_AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/recommendation_CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(RECOMMENDATION_ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/recommendation_CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FALSE_POSITIVE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/false_positive.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/empty.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
@@ -1572,16 +1354,200 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
|
||||
|
||||
private void loadTypeForTest() {
|
||||
|
||||
typeColorMap.put(VERTEBRATE, "#ff85f7");
|
||||
typeColorMap.put(ADDRESS, "#ffe187");
|
||||
typeColorMap.put(AUTHOR, "#ffe187");
|
||||
typeColorMap.put(SPONSOR, "#85ebff");
|
||||
typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
|
||||
typeColorMap.put(REDACTION_INDICATOR, "#caff85");
|
||||
typeColorMap.put(HINT_ONLY, "#abc0c4");
|
||||
typeColorMap.put(MUST_REDACT, "#fab4c0");
|
||||
typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
|
||||
typeColorMap.put(TEST_METHOD, "#91fae8");
|
||||
typeColorMap.put(PII, "#66ccff");
|
||||
typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
|
||||
typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
|
||||
typeColorMap.put(FALSE_POSITIVE, "#ffffff");
|
||||
typeColorMap.put(PURITY, "#ffe187");
|
||||
typeColorMap.put(IMAGE, "#fcc5fb");
|
||||
typeColorMap.put(OCR, "#fcc5fb");
|
||||
typeColorMap.put(LOGO, "#ffe187");
|
||||
typeColorMap.put(FORMULA, "#ffe187");
|
||||
typeColorMap.put(SIGNATURE, "#ffe187");
|
||||
typeColorMap.put(IMPORTED_REDACTION, "#32a852");
|
||||
|
||||
hintTypeMap.put(VERTEBRATE, true);
|
||||
hintTypeMap.put(ADDRESS, false);
|
||||
hintTypeMap.put(AUTHOR, false);
|
||||
hintTypeMap.put(SPONSOR, false);
|
||||
hintTypeMap.put(NO_REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(HINT_ONLY, true);
|
||||
hintTypeMap.put(MUST_REDACT, true);
|
||||
hintTypeMap.put(PUBLISHED_INFORMATION, true);
|
||||
hintTypeMap.put(TEST_METHOD, true);
|
||||
hintTypeMap.put(PII, false);
|
||||
hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
hintTypeMap.put(FALSE_POSITIVE, true);
|
||||
hintTypeMap.put(PURITY, false);
|
||||
hintTypeMap.put(IMAGE, true);
|
||||
hintTypeMap.put(OCR, true);
|
||||
hintTypeMap.put(FORMULA, false);
|
||||
hintTypeMap.put(LOGO, false);
|
||||
hintTypeMap.put(SIGNATURE, false);
|
||||
hintTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
hintTypeMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
caseInSensitiveMap.put(VERTEBRATE, true);
|
||||
caseInSensitiveMap.put(ADDRESS, false);
|
||||
caseInSensitiveMap.put(AUTHOR, false);
|
||||
caseInSensitiveMap.put(SPONSOR, false);
|
||||
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(HINT_ONLY, true);
|
||||
caseInSensitiveMap.put(MUST_REDACT, true);
|
||||
caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
|
||||
caseInSensitiveMap.put(TEST_METHOD, false);
|
||||
caseInSensitiveMap.put(PII, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
caseInSensitiveMap.put(FALSE_POSITIVE, false);
|
||||
caseInSensitiveMap.put(PURITY, false);
|
||||
caseInSensitiveMap.put(IMAGE, true);
|
||||
caseInSensitiveMap.put(OCR, true);
|
||||
caseInSensitiveMap.put(SIGNATURE, true);
|
||||
caseInSensitiveMap.put(LOGO, true);
|
||||
caseInSensitiveMap.put(FORMULA, true);
|
||||
caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
|
||||
caseInSensitiveMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
recommendationTypeMap.put(VERTEBRATE, false);
|
||||
recommendationTypeMap.put(ADDRESS, false);
|
||||
recommendationTypeMap.put(AUTHOR, false);
|
||||
recommendationTypeMap.put(SPONSOR, false);
|
||||
recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(HINT_ONLY, false);
|
||||
recommendationTypeMap.put(MUST_REDACT, false);
|
||||
recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
|
||||
recommendationTypeMap.put(TEST_METHOD, false);
|
||||
recommendationTypeMap.put(PII, false);
|
||||
recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
|
||||
recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
|
||||
recommendationTypeMap.put(FALSE_POSITIVE, false);
|
||||
recommendationTypeMap.put(PURITY, false);
|
||||
recommendationTypeMap.put(IMAGE, false);
|
||||
recommendationTypeMap.put(OCR, false);
|
||||
recommendationTypeMap.put(FORMULA, false);
|
||||
recommendationTypeMap.put(SIGNATURE, false);
|
||||
recommendationTypeMap.put(LOGO, false);
|
||||
recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
recommendationTypeMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
rankTypeMap.put(FALSE_POSITIVE, 160);
|
||||
rankTypeMap.put(PURITY, 155);
|
||||
rankTypeMap.put(PII, 150);
|
||||
rankTypeMap.put(ADDRESS, 140);
|
||||
rankTypeMap.put(AUTHOR, 130);
|
||||
rankTypeMap.put(SPONSOR, 120);
|
||||
rankTypeMap.put(VERTEBRATE, 110);
|
||||
rankTypeMap.put(MUST_REDACT, 100);
|
||||
rankTypeMap.put(REDACTION_INDICATOR, 90);
|
||||
rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
|
||||
rankTypeMap.put(PUBLISHED_INFORMATION, 70);
|
||||
rankTypeMap.put(TEST_METHOD, 60);
|
||||
rankTypeMap.put(HINT_ONLY, 50);
|
||||
rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
|
||||
rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
|
||||
rankTypeMap.put(IMAGE, 30);
|
||||
rankTypeMap.put(OCR, 29);
|
||||
rankTypeMap.put(LOGO, 28);
|
||||
rankTypeMap.put(SIGNATURE, 27);
|
||||
rankTypeMap.put(FORMULA, 26);
|
||||
rankTypeMap.put(DOSSIER_REDACTIONS, 200);
|
||||
rankTypeMap.put(IMPORTED_REDACTION, 200);
|
||||
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
colors.setRequestAdd("#04b093");
|
||||
colors.setRequestRemove("#04b093");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
}
|
||||
|
||||
|
||||
private List<Type> getTypeResponse() {
|
||||
|
||||
return typeColorMap.entrySet()
|
||||
.stream()
|
||||
.map(typeColor -> Type.builder()
|
||||
.id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.type(typeColor.getKey())
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColor.getValue())
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
|
||||
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
|
||||
.rank(rankTypeMap.get(typeColor.getKey()))
|
||||
.build())
|
||||
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
|
||||
|
||||
return Type.builder()
|
||||
.id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColorMap.get(type))
|
||||
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
|
||||
.isHint(hintTypeMap.get(type))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(type))
|
||||
.isRecommendation(recommendationTypeMap.get(type))
|
||||
.rank(rankTypeMap.get(type))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private String cleanDictionaryEntry(String entry) {
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
|
||||
}
|
||||
|
||||
|
||||
private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
|
||||
|
||||
List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
|
||||
entries.forEach(entry -> {
|
||||
dictionaryEntries.add(DictionaryEntry.builder()
|
||||
.value(entry)
|
||||
.version(reanlysisVersions.getOrDefault(entry, 0L))
|
||||
.deleted(deleted.contains(entry))
|
||||
.build());
|
||||
});
|
||||
return dictionaryEntries;
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testImportedRedactions() throws IOException {
|
||||
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/S13.pdf");
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/S13.json");
|
||||
List<ImportedAnnotation> importedAnnotations = objectMapper.readValue(importedRedactions.getInputStream().readAllBytes(), new TypeReference<List<ImportedAnnotation>>() {
|
||||
});
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/ImportedRedactions.pdf");
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/ImportedRedactions.json");
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
request.setImportedAnnotations(importedAnnotations);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
@@ -1598,4 +1564,32 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(InputStream stream) {
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
|
||||
return request;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private AnalyzeRequest prepareStorage(String file) {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(file);
|
||||
|
||||
return prepareStorage(pdfFileResource.getInputStream());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ platform.multi-tenancy:
|
||||
|
||||
redaction-service:
|
||||
enable-image-classification: false
|
||||
enable-entity-recognition: false
|
||||
enable-entity-recognition: true
|
||||
|
||||
storage:
|
||||
signer-type: 'AWSS3V4SignerType'
|
||||
|
||||
+6
@@ -3,3 +3,9 @@ Report Number: 33168
|
||||
Page
|
||||
Report Number: BFI0714
|
||||
Tesh Consultants International
|
||||
B. Rahim
|
||||
C. J. Alfred
|
||||
Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP
|
||||
Özgür U. Reyhan
|
||||
Sude Halide Nurullah
|
||||
Xinyi Y. Tao
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"importedRedactions":{"1":[{"id":"755273b5ddc6933eff7f6b6e055c803f","positions":[{"topLeft":{"x":275.0,"y":520.504},"width":55.675,"height":15.582,"page":1}]},{"id":"5e517d2c890272fc140578a26971b459","positions":[{"topLeft":{"x":164.0,"y":316.492},"width":50.7,"height":15.658,"page":1},{"topLeft":{"x":127.0,"y":302.613},"width":119.675,"height":14.902,"page":1},{"topLeft":{"x":81.0,"y":289.622},"width":210.025,"height":14.846,"page":1},{"topLeft":{"x":136.0,"y":275.912},"width":94.2,"height":13.033,"page":1},{"topLeft":{"x":158.0,"y":261.402},"width":52.793,"height":16.218,"page":1},{"topLeft":{"x":177.0,"y":247.413},"width":15.507,"height":16.148,"page":1}]}]}}
|
||||
BIN
Binary file not shown.
-121
@@ -1,121 +0,0 @@
|
||||
[
|
||||
{
|
||||
"annotationId": "6113d300d4a5cd7e0e91ef692e046fc5",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 519.11,
|
||||
"width": 122.496,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "04d043c73922f6e0dc50a1c7484e9425",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 544.43,
|
||||
"width": 83.414,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "83f5f2cbe55f2acbbf7adc479cf694cb",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 557.03,
|
||||
"width": 57.624,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "4bacdb3675c46181c75295fe8c6e6562",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 531.83,
|
||||
"width": 83.414,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "8db925e788f2ab1c9ded097a9de10111",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 148.94,
|
||||
"topLeftY": 317.49,
|
||||
"width": 141.263,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "a4d2f29216efe7507bf97d2e55010dd4",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 148.94,
|
||||
"topLeftY": 342.81,
|
||||
"width": 94.389,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "9983b5000e8f3540a402178f91b3a2e2",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 148.94,
|
||||
"topLeftY": 330.21,
|
||||
"width": 94.332,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "APPROVED",
|
||||
"user": "its a me",
|
||||
"comment": "First Test",
|
||||
"processedDate": "2022-01-27T12:57:25.48Z"
|
||||
}
|
||||
]
|
||||
BIN
Binary file not shown.
+872
@@ -0,0 +1,872 @@
|
||||
{
|
||||
"result": {
|
||||
"1": [
|
||||
{
|
||||
"value": "Lastname, J.",
|
||||
"startOffset": 54,
|
||||
"endOffset": 66,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Doe, M.",
|
||||
"startOffset": 67,
|
||||
"endOffset": 74,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Mustermann Lastname M.",
|
||||
"startOffset": 75,
|
||||
"endOffset": 97,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Doe J. Mustermann M.",
|
||||
"startOffset": 99,
|
||||
"endOffset": 119,
|
||||
"type": "CBI_author"
|
||||
}
|
||||
],
|
||||
"2": [
|
||||
{
|
||||
"value": "Eikenboom Charalampos",
|
||||
"startOffset": 148,
|
||||
"endOffset": 169,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Schenk Tanja Schmitt ←",
|
||||
"startOffset": 170,
|
||||
"endOffset": 192,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"3": [
|
||||
{
|
||||
"value": "Rue Jean Baffier",
|
||||
"startOffset": 214,
|
||||
"endOffset": 230,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "7232",
|
||||
"startOffset": 155,
|
||||
"endOffset": 159,
|
||||
"type": "POSTAL"
|
||||
},
|
||||
{
|
||||
"value": "CX",
|
||||
"startOffset": 160,
|
||||
"endOffset": 162,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Warnsveld",
|
||||
"startOffset": 163,
|
||||
"endOffset": 172,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Netherlands",
|
||||
"startOffset": 174,
|
||||
"endOffset": 185,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "NL",
|
||||
"startOffset": 187,
|
||||
"endOffset": 189,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Institut Industries",
|
||||
"startOffset": 190,
|
||||
"endOffset": 209,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "33",
|
||||
"startOffset": 211,
|
||||
"endOffset": 213,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Rue Jean Baffier",
|
||||
"startOffset": 214,
|
||||
"endOffset": 230,
|
||||
"type": "STREET"
|
||||
},
|
||||
{
|
||||
"value": "18000",
|
||||
"startOffset": 232,
|
||||
"endOffset": 237,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Bourges",
|
||||
"startOffset": 238,
|
||||
"endOffset": 245,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "France",
|
||||
"startOffset": 247,
|
||||
"endOffset": 253,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "18300",
|
||||
"startOffset": 282,
|
||||
"endOffset": 287,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Saint-Satur",
|
||||
"startOffset": 288,
|
||||
"endOffset": 299,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "France",
|
||||
"startOffset": 301,
|
||||
"endOffset": 307,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Lesdo Industries",
|
||||
"startOffset": 312,
|
||||
"endOffset": 328,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Chäppelisträssli",
|
||||
"startOffset": 330,
|
||||
"endOffset": 346,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "6078",
|
||||
"startOffset": 348,
|
||||
"endOffset": 352,
|
||||
"type": "POSTAL"
|
||||
},
|
||||
{
|
||||
"value": "Lungern",
|
||||
"startOffset": 353,
|
||||
"endOffset": 360,
|
||||
"type": "STREET"
|
||||
},
|
||||
{
|
||||
"value": "Switzerland",
|
||||
"startOffset": 362,
|
||||
"endOffset": 373,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Shlissel'burgskaya Ulitsa",
|
||||
"startOffset": 374,
|
||||
"endOffset": 399,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Nizhny Novgorod Oblast",
|
||||
"startOffset": 401,
|
||||
"endOffset": 423,
|
||||
"type": "STREET"
|
||||
},
|
||||
{
|
||||
"value": "Russia",
|
||||
"startOffset": 425,
|
||||
"endOffset": 431,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "603034",
|
||||
"startOffset": 433,
|
||||
"endOffset": 439,
|
||||
"type": "POSTAL"
|
||||
},
|
||||
{
|
||||
"value": "RU",
|
||||
"startOffset": 441,
|
||||
"endOffset": 443,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Karl Johans Gate",
|
||||
"startOffset": 444,
|
||||
"endOffset": 460,
|
||||
"type": "STREET"
|
||||
},
|
||||
{
|
||||
"value": "11",
|
||||
"startOffset": 461,
|
||||
"endOffset": 463,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "0154",
|
||||
"startOffset": 465,
|
||||
"endOffset": 469,
|
||||
"type": "POSTAL"
|
||||
},
|
||||
{
|
||||
"value": "Oslo",
|
||||
"startOffset": 470,
|
||||
"endOffset": 474,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Norway",
|
||||
"startOffset": 476,
|
||||
"endOffset": 482,
|
||||
"type": "COUNTRY"
|
||||
}
|
||||
],
|
||||
"4": [
|
||||
{
|
||||
"value": "Expand",
|
||||
"startOffset": 67,
|
||||
"endOffset": 73,
|
||||
"type": "STATE"
|
||||
},
|
||||
{
|
||||
"value": "Hint Clarissa",
|
||||
"startOffset": 77,
|
||||
"endOffset": 90,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Dict",
|
||||
"startOffset": 114,
|
||||
"endOffset": 118,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Authors-Dict",
|
||||
"startOffset": 171,
|
||||
"endOffset": 183,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"6": [
|
||||
{
|
||||
"value": "Michael N.",
|
||||
"startOffset": 7,
|
||||
"endOffset": 17,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Weyland Industries",
|
||||
"startOffset": 76,
|
||||
"endOffset": 94,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"7": [
|
||||
{
|
||||
"value": "Funnarie B.",
|
||||
"startOffset": 7,
|
||||
"endOffset": 18,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Authentic Diagnostics",
|
||||
"startOffset": 73,
|
||||
"endOffset": 94,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"8": [
|
||||
{
|
||||
"value": "Feuer A.",
|
||||
"startOffset": 6,
|
||||
"endOffset": 14,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Tyrell Corporation",
|
||||
"startOffset": 74,
|
||||
"endOffset": 92,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"12": [
|
||||
{
|
||||
"value": "Melanie",
|
||||
"startOffset": 292,
|
||||
"endOffset": 299,
|
||||
"type": "CBI_author"
|
||||
}
|
||||
],
|
||||
"13": [
|
||||
{
|
||||
"value": "Stark Industries",
|
||||
"startOffset": 184,
|
||||
"endOffset": 200,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"14": [
|
||||
{
|
||||
"value": "Omni Consumer Products Do",
|
||||
"startOffset": 197,
|
||||
"endOffset": 222,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"15": [
|
||||
{
|
||||
"value": "Omni Consumer Products Do",
|
||||
"startOffset": 122,
|
||||
"endOffset": 147,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"16": [
|
||||
{
|
||||
"value": "Asya Lyon",
|
||||
"startOffset": 253,
|
||||
"endOffset": 262,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Carina Madsen",
|
||||
"startOffset": 264,
|
||||
"endOffset": 277,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Alexandra Häusler",
|
||||
"startOffset": 279,
|
||||
"endOffset": 296,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Hanke Mendel",
|
||||
"startOffset": 298,
|
||||
"endOffset": 310,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Kwok, Jun K.",
|
||||
"startOffset": 444,
|
||||
"endOffset": 456,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Tu Wong",
|
||||
"startOffset": 458,
|
||||
"endOffset": 465,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Qiang Suen",
|
||||
"startOffset": 467,
|
||||
"endOffset": 477,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Zhou Mah",
|
||||
"startOffset": 479,
|
||||
"endOffset": 487,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Lei W. Huang",
|
||||
"startOffset": 499,
|
||||
"endOffset": 511,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Ru X.",
|
||||
"startOffset": 513,
|
||||
"endOffset": 518,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Oxford University Press",
|
||||
"startOffset": 166,
|
||||
"endOffset": 189,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Iakovos Geiger",
|
||||
"startOffset": 222,
|
||||
"endOffset": 236,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Julian Ritter",
|
||||
"startOffset": 238,
|
||||
"endOffset": 251,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Asya Lyon",
|
||||
"startOffset": 253,
|
||||
"endOffset": 262,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Carina Madsen",
|
||||
"startOffset": 264,
|
||||
"endOffset": 277,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Alexandra Häusler",
|
||||
"startOffset": 279,
|
||||
"endOffset": 296,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Hanke Mendel",
|
||||
"startOffset": 298,
|
||||
"endOffset": 310,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Ranya",
|
||||
"startOffset": 312,
|
||||
"endOffset": 317,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Eikenboom",
|
||||
"startOffset": 318,
|
||||
"endOffset": 327,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Min Kwok",
|
||||
"startOffset": 440,
|
||||
"endOffset": 448,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Qiang Suen",
|
||||
"startOffset": 467,
|
||||
"endOffset": 477,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Zhou Mah",
|
||||
"startOffset": 479,
|
||||
"endOffset": 487,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Ning Liu",
|
||||
"startOffset": 489,
|
||||
"endOffset": 497,
|
||||
"type": "STREET"
|
||||
},
|
||||
{
|
||||
"value": "Lei W. Huang, Ru X. Wu",
|
||||
"startOffset": 499,
|
||||
"endOffset": 521,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"17": [
|
||||
{
|
||||
"value": "Nurullah Özgür",
|
||||
"startOffset": 210,
|
||||
"endOffset": 224,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Reyhan B.",
|
||||
"startOffset": 228,
|
||||
"endOffset": 237,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Alfred Xinyi Y.",
|
||||
"startOffset": 250,
|
||||
"endOffset": 265,
|
||||
"type": "CBI_author"
|
||||
},
|
||||
{
|
||||
"value": "Redacted",
|
||||
"startOffset": 12,
|
||||
"endOffset": 20,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Aomachi",
|
||||
"startOffset": 154,
|
||||
"endOffset": 161,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Nomi",
|
||||
"startOffset": 163,
|
||||
"endOffset": 167,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Ishikawa",
|
||||
"startOffset": 169,
|
||||
"endOffset": 177,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "923-1101",
|
||||
"startOffset": 178,
|
||||
"endOffset": 186,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Japan",
|
||||
"startOffset": 188,
|
||||
"endOffset": 193,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "JP",
|
||||
"startOffset": 195,
|
||||
"endOffset": 197,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Sude Halide Nurullah Özgür U. Reyhan B. Rahim C. J. Alfred Xinyi Y. Tao Clara Siegfried",
|
||||
"startOffset": 198,
|
||||
"endOffset": 285,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Dict",
|
||||
"startOffset": 301,
|
||||
"endOffset": 305,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"18": [
|
||||
{
|
||||
"value": "Redact Emails",
|
||||
"startOffset": 12,
|
||||
"endOffset": 25,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"19": [
|
||||
{
|
||||
"value": "Central Research Industry",
|
||||
"startOffset": 274,
|
||||
"endOffset": 299,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Maximiliam Schmitt",
|
||||
"startOffset": 530,
|
||||
"endOffset": 548,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "European Central Institute",
|
||||
"startOffset": 727,
|
||||
"endOffset": 753,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Emilia Lockhart Alternative",
|
||||
"startOffset": 775,
|
||||
"endOffset": 802,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Cyberdyne Systems Tower",
|
||||
"startOffset": 812,
|
||||
"endOffset": 835,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "121a",
|
||||
"startOffset": 844,
|
||||
"endOffset": 848,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Hong Kong",
|
||||
"startOffset": 849,
|
||||
"endOffset": 858,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "BT",
|
||||
"startOffset": 860,
|
||||
"endOffset": 862,
|
||||
"type": "COUNTRY"
|
||||
}
|
||||
],
|
||||
"21": [
|
||||
{
|
||||
"value": "Central Research Industry",
|
||||
"startOffset": 270,
|
||||
"endOffset": 295,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Maximiliam Schmitt",
|
||||
"startOffset": 526,
|
||||
"endOffset": 544,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "European Central Institute",
|
||||
"startOffset": 723,
|
||||
"endOffset": 749,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Emilia Lockhart Alternative",
|
||||
"startOffset": 771,
|
||||
"endOffset": 798,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Cyberdyne Systems Tower",
|
||||
"startOffset": 808,
|
||||
"endOffset": 831,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "121a",
|
||||
"startOffset": 840,
|
||||
"endOffset": 844,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Hong Kong",
|
||||
"startOffset": 845,
|
||||
"endOffset": 854,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "BT",
|
||||
"startOffset": 856,
|
||||
"endOffset": 858,
|
||||
"type": "COUNTRY"
|
||||
}
|
||||
],
|
||||
"22": [
|
||||
{
|
||||
"value": "Riddley",
|
||||
"startOffset": 236,
|
||||
"endOffset": 243,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "359-21",
|
||||
"startOffset": 259,
|
||||
"endOffset": 265,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Huam-dong",
|
||||
"startOffset": 266,
|
||||
"endOffset": 275,
|
||||
"type": "STATE"
|
||||
},
|
||||
{
|
||||
"value": "Yongsan-gu",
|
||||
"startOffset": 276,
|
||||
"endOffset": 286,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Seoul",
|
||||
"startOffset": 287,
|
||||
"endOffset": 292,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "South Korea Phone",
|
||||
"startOffset": 294,
|
||||
"endOffset": 311,
|
||||
"type": "COUNTRY"
|
||||
},
|
||||
{
|
||||
"value": "Soylent Corporation",
|
||||
"startOffset": 500,
|
||||
"endOffset": 519,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"23": [
|
||||
{
|
||||
"value": "Umbrella Corporation",
|
||||
"startOffset": 208,
|
||||
"endOffset": 228,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Jill",
|
||||
"startOffset": 238,
|
||||
"endOffset": 242,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "359-21",
|
||||
"startOffset": 262,
|
||||
"endOffset": 268,
|
||||
"type": "CARDINAL"
|
||||
},
|
||||
{
|
||||
"value": "Huam-dong",
|
||||
"startOffset": 269,
|
||||
"endOffset": 278,
|
||||
"type": "STATE"
|
||||
},
|
||||
{
|
||||
"value": "Yongsan-gu",
|
||||
"startOffset": 279,
|
||||
"endOffset": 289,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "Seoul",
|
||||
"startOffset": 290,
|
||||
"endOffset": 295,
|
||||
"type": "CITY"
|
||||
},
|
||||
{
|
||||
"value": "South Korea Phone",
|
||||
"startOffset": 297,
|
||||
"endOffset": 314,
|
||||
"type": "COUNTRY"
|
||||
}
|
||||
],
|
||||
"26": [
|
||||
{
|
||||
"value": "Umbrella Corporation",
|
||||
"startOffset": 209,
|
||||
"endOffset": 229,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"28": [
|
||||
{
|
||||
"value": "Purity Hint",
|
||||
"startOffset": 9,
|
||||
"endOffset": 20,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Hint",
|
||||
"startOffset": 35,
|
||||
"endOffset": 39,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"29": [
|
||||
{
|
||||
"value": "Ignore",
|
||||
"startOffset": 9,
|
||||
"endOffset": 15,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"30": [
|
||||
{
|
||||
"value": "Redact Signatures Redact",
|
||||
"startOffset": 12,
|
||||
"endOffset": 36,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Dilara",
|
||||
"startOffset": 139,
|
||||
"endOffset": 145,
|
||||
"type": "ORG"
|
||||
},
|
||||
{
|
||||
"value": "Tobias Müller",
|
||||
"startOffset": 197,
|
||||
"endOffset": 210,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"31": [
|
||||
{
|
||||
"value": "Redact Logo Redact",
|
||||
"startOffset": 9,
|
||||
"endOffset": 27,
|
||||
"type": "ORG"
|
||||
}
|
||||
],
|
||||
"41": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"42": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"43": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"44": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"45": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"46": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"47": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"48": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
],
|
||||
"49": [
|
||||
{
|
||||
"value": "Page-Footer",
|
||||
"startOffset": 10,
|
||||
"endOffset": 21,
|
||||
"type": "STREET"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"result": {
|
||||
"1": [
|
||||
{
|
||||
"value": "Mannheim",
|
||||
"startOffset": 0,
|
||||
"endOffset": 8,
|
||||
"type": "CITY"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
Reference in New Issue
Block a user