Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* fix NerEntities CBI_address combination
This commit is contained in:
Kilian Schuettler 2023-04-28 16:37:51 +02:00
parent f891e02935
commit 37a3ee8d54
6 changed files with 1016 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
View File

@ -1,7 +1,9 @@
package com.iqser.red.service.redaction.v1.server.redaction.adapter; package com.iqser.red.service.redaction.v1.server.redaction.adapter;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -9,48 +11,32 @@ import java.util.stream.Stream;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@Service @Service
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class NerEntitiesAdapter { public class NerEntitiesAdapter {
RedactionStorageService redactionStorageService;
RedactionServiceSettings redactionServiceSettings;
// imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition // imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition
private static final Set<String> CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE"); static Set<String> CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE");
private static final Set<String> CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY"); static Set<String> CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY");
private static final int MAX_DISTANCE_BETWEEN_PARTS = 20;
private static final int MIN_PART_MATCHES = 3; static int MAX_DISTANCE_BETWEEN_PARTS = 20;
private static final boolean ALLOW_DUPLICATES = false; static int MIN_PART_MATCHES = 3;
static boolean ALLOW_DUPLICATES;
public List<EntityRecognitionEntity> getNerEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) { public List<EntityRecognitionEntity> validateAndCombine(NerEntities nerEntities, DocumentGraph documentGraph) {
NerEntities nerEntities; List<EntityRecognitionEntity> entityRecognitionEntities = getValidatedEntities(nerEntities, documentGraph).toList();
if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
} else {
nerEntities = new NerEntities();
}
addSectionStartOffsetToEachNerEntity(getStringStartOffsetsForMainSections(documentGraph), nerEntities);
List<EntityRecognitionEntity> entityRecognitionEntities = validateForCorrectOffsets(documentGraph, nerEntities);
List<EntityRecognitionEntity> cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList(); List<EntityRecognitionEntity> cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList();
List<EntityRecognitionEntity> cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock()); List<EntityRecognitionEntity> cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock());
@ -59,26 +45,65 @@ public class NerEntitiesAdapter {
} }
protected Stream<EntityRecognitionEntity> getValidatedEntities(NerEntities nerEntities, DocumentGraph documentGraph) {
return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntities) //
.filter(nerEntity -> nerEntityOffsetMatchesDocumentGraph(nerEntity, documentGraph.buildTextBlock()));
}
private List<EntityRecognitionEntity> combineToCBIAddressEntities(List<EntityRecognitionEntity> entityRecognitionEntities, TextBlock textBlock) { private List<EntityRecognitionEntity> combineToCBIAddressEntities(List<EntityRecognitionEntity> entityRecognitionEntities, TextBlock textBlock) {
Set<EntityRecognitionEntity> entitiesOfTypeCBIAddressParts = entityRecognitionEntities.stream() List<EntityRecognitionEntity> sortedEntities = entityRecognitionEntities.stream()
.filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType())) .filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType()))
.collect(Collectors.toSet()); .sorted(Comparator.comparingInt(EntityRecognitionEntity::getStartOffset))
.toList();
return entitiesOfTypeCBIAddressParts.stream() if (sortedEntities.isEmpty()) {
.filter(entity -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(entity.getType())) return sortedEntities;
.map(entity -> findFollowingEntitiesOfTypeCBIAddressPart(entity, entitiesOfTypeCBIAddressParts)) }
.filter(entities -> entities.size() < MIN_PART_MATCHES)
.map(this::toContainingBoundary) List<List<EntityRecognitionEntity>> entityClusters = new LinkedList<>();
.distinct()
List<EntityRecognitionEntity> startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(e.getType())).toList();
for (EntityRecognitionEntity startEntity : startEntitiesOfEssentialType) {
List<EntityRecognitionEntity> currentCluster = new LinkedList<>();
entityClusters.add(currentCluster);
int lastEndOffset = startEntity.getStartOffset();
for (EntityRecognitionEntity entity : sortedEntities) {
if (entity.getStartOffset() < lastEndOffset) {
continue;
}
if (disctanceIsLargerThanMaxDistance(lastEndOffset, entity) || isDuplicate(currentCluster, entity)) {
currentCluster = new LinkedList<>();
entityClusters.add(currentCluster);
currentCluster.add(entity);
lastEndOffset = entity.getEndOffset();
} else {
currentCluster.add(entity);
lastEndOffset = entity.getEndOffset();
}
}
}
List<Boundary> boundaries = entityClusters.stream().filter(cluster -> cluster.size() >= MIN_PART_MATCHES).map(this::toContainingBoundary).distinct().toList();
return boundaries.stream()
.map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address")) .map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address"))
.toList(); .toList();
} }
private List<EntityRecognitionEntity> validateForCorrectOffsets(DocumentGraph documentGraph, NerEntities nerEntities) { private static boolean isDuplicate(List<EntityRecognitionEntity> currentCluster, EntityRecognitionEntity entity) {
return nerEntities.getData().values().stream().flatMap(Collection::stream).filter(nerEntity -> nerEntityOffsetMatches(nerEntity, documentGraph.buildTextBlock())).toList(); return ALLOW_DUPLICATES || currentCluster.stream().anyMatch(e -> e.getType().equals(entity.getType()));
}
private static boolean disctanceIsLargerThanMaxDistance(int lastEndOffset, EntityRecognitionEntity entity) {
return (entity.getStartOffset() - lastEndOffset) > MAX_DISTANCE_BETWEEN_PARTS;
} }
@ -122,19 +147,20 @@ public class NerEntitiesAdapter {
} }
private boolean nerEntityOffsetMatches(EntityRecognitionEntity nerEntity, TextBlock textBlock) { private boolean nerEntityOffsetMatchesDocumentGraph(EntityRecognitionEntity nerEntity, TextBlock textBlock) {
return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset())); return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset()));
} }
private static void addSectionStartOffsetToEachNerEntity(List<Integer> stringOffsetsForMainSections, NerEntities nerEntities) { private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntities nerEntities) {
nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> { nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {
int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key); int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key);
entityRecognitionEntity.setStartOffset(newStartOffset); entityRecognitionEntity.setStartOffset(newStartOffset);
entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
})); }));
return nerEntities.getData().values().stream().flatMap(Collection::stream);
} }

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE; import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -166,7 +167,12 @@ public class AnalyzeService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode)); sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode));
List<EntityRecognitionEntity> nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph); List<EntityRecognitionEntity> nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
} else {
nerEntities = Collections.emptyList();
}
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
@ -198,7 +204,13 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
List<EntityRecognitionEntity> nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph); List<EntityRecognitionEntity> nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
} else {
nerEntities = Collections.emptyList();
}
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());

33
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphIntegrationTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.graph; package com.iqser.red.service.redaction.v1.server.document.graph;
import java.io.InputStream; import java.io.InputStream;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -61,12 +62,15 @@ public class BuildDocumentGraphIntegrationTest extends AbstractRedactionIntegrat
@SneakyThrows @SneakyThrows
protected DocumentGraph buildGraph(String filename) { protected DocumentGraph buildGraph(String filename) {
if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) { if (!filename.endsWith(".pdf")) {
prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json"); filename = filename + ".pdf";
} else {
uploadFileToStorage(filename + ".pdf");
} }
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
prepareStorage(filename, "files/cv_service_empty_response.json", filename.replace(".pdf", ".IMAGE_INFO.json"));
} else {
uploadFileToStorage(filename);
}
ClassPathResource fileResource = new ClassPathResource(filename);
try (InputStream inputStream = fileResource.getInputStream()) { try (InputStream inputStream = fileResource.getInputStream()) {
Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID); Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
@ -75,4 +79,23 @@ public class BuildDocumentGraphIntegrationTest extends AbstractRedactionIntegrat
} }
} }
@SneakyThrows
protected DocumentGraph buildGraphNoImages(String filename) {
if (!filename.endsWith(".pdf")) {
filename = filename + ".pdf";
}
uploadFileToStorage(filename);
ClassPathResource fileResource = new ClassPathResource(filename);
try (InputStream inputStream = fileResource.getInputStream()) {
Map<Integer, List<ClassifiedImage>> pdfImages = new HashMap<>();
Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
return DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
}
}
} }

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionIntegrationTest.java
View File

@ -33,7 +33,7 @@ public class DocumentGraphEntityInsertionIntegrationTest extends BuildDocumentGr
@Test @Test
public void assertCollectAllEntitiesWorks() { public void assertCollectAllEntitiesWorks() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document"); DocumentGraph documentGraph = buildGraph("files/new/crafted document.pdf");
createAndInsertEntity(documentGraph, "Clarissa"); createAndInsertEntity(documentGraph, "Clarissa");
createAndInsertEntity(documentGraph, "Lastname"); createAndInsertEntity(documentGraph, "Lastname");
createAndInsertEntity(documentGraph, "David Ksenia"); createAndInsertEntity(documentGraph, "David Ksenia");

119
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java Normal file
View File

@ -0,0 +1,119 @@
package com.iqser.red.service.redaction.v1.server.redaction.adapter;
import static org.wildfly.common.Assert.assertFalse;
import static org.wildfly.common.Assert.assertTrue;
import java.awt.Color;
import java.awt.geom.Rectangle2D;
import java.io.File;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
import lombok.SneakyThrows;
class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
@Autowired
private NerEntitiesAdapter nerEntitiesAdapter;
@Autowired
private EntityCreationService entityCreationService;
@SneakyThrows
private static NerEntities parseNerEntities(String filePath) {
ClassPathResource resource = new ClassPathResource(filePath);
return ObjectMapperFactory.create().readValue(resource.getInputStream(), NerEntities.class);
}
@Test
@SneakyThrows
public void testGetNerEntities() {
String filePath = "files/new/crafted document.pdf";
String nerEntitiesFilePath = "ner_entities/crafted document.NER_ENTITIES.json";
DocumentGraph documentGraph = buildGraphNoImages(filePath);
List<EntityRecognitionEntity> entityRecognitionEntities = nerEntitiesAdapter.validateAndCombine(parseNerEntities(nerEntitiesFilePath), documentGraph);
assertFalse(entityRecognitionEntities.isEmpty());
assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.getStartOffset() < entity.getEndOffset()));
ClassPathResource resource = new ClassPathResource(filePath);
PDDocument pdDocument = PDDocument.load(resource.getInputStream());
Stream<EntityRecognitionEntity> unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
.filter(e -> !e.getType().equals("CBI_author"));
List<RedactionEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
.map(e -> entityCreationService.byBoundary(new Boundary(e.getStartOffset(), e.getEndOffset()), e.getType(), EntityType.ENTITY, documentGraph))
.toList();
redactionEntities.stream()
.collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))
.forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber,
getPositionsFromEntityOfType("CBI_author", entities),
getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities),
getPositionsFromEntityOfType("CBI_address", entities),
pdDocument));
File outputFile = new File("/tmp/nerEntities.pdf");
pdDocument.save(outputFile);
pdDocument.close();
}
private List<Rectangle2D> getPositionsFromEntities(Stream<RedactionEntity> entities) {
return entities.map(RedactionEntity::getRedactionPositionsPerPage)
.flatMap(Collection::stream)
.map(RedactionPosition::getRectanglePerLine)
.flatMap(Collection::stream)
.toList();
}
private List<Rectangle2D> getPositionsFromEntityOfType(String type, List<RedactionEntity> entities) {
return getPositionsFromEntities(entities.stream().filter(e -> e.getType().equals(type)));
}
private List<Rectangle2D> getPositionsFromEntityNotOfType(List<String> types, List<RedactionEntity> entities) {
return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.getType().equals(type))));
}
@SneakyThrows
private static void drawNerEntitiesAsPartsAndCombined(int pageNumber,
List<Rectangle2D> cbiAuthorRects,
List<Rectangle2D> addressPartsRects,
List<Rectangle2D> cbiAddressRects,
PDDocument pdDocument) {
//PdfDraw.drawRectangle2DList(pdDocument, pageNumber, cbiAuthorRects, PdfDraw.Options.builder().stroke(true).strokeColor(Color.blue).build());
PdfDraw.drawRectangle2DList(pdDocument, pageNumber, addressPartsRects, PdfDraw.Options.builder().stroke(true).strokeColor(Color.MAGENTA).build());
PdfDraw.drawRectangle2DList(pdDocument, pageNumber, cbiAddressRects, PdfDraw.Options.builder().stroke(true).strokeColor(Color.green).build());
}
}

792
redaction-service-v1/redaction-service-server-v1/src/test/resources/ner_entities/crafted document.NER_ENTITIES.json Normal file
View File

@ -0,0 +1,792 @@
{
"dossierId": "ef6d4541-4fea-4451-932a-55d167ab30dc",
"fileId": "44a0ab4b930ac6bd89dbbfa7c23f9c98",
"targetFileExtension": "SIMPLIFIED_TEXT.json.gz",
"responseFileExtension": "NER_ENTITIES.json.gz",
"data": {
"0": [
{
"value": "Lastname, J.",
"startOffset": 54,
"endOffset": 66,
"type": "CBI_author"
},
{
"value": "Doe, M.",
"startOffset": 67,
"endOffset": 74,
"type": "CBI_author"
},
{
"value": "Mustermann Lastname M.",
"startOffset": 75,
"endOffset": 97,
"type": "CBI_author"
},
{
"value": "Doe J. Mustermann M.",
"startOffset": 99,
"endOffset": 119,
"type": "CBI_author"
}
],
"1": [
{
"value": "Eikenboom Charalampos",
"startOffset": 148,
"endOffset": 169,
"type": "ORG"
},
{
"value": "Schenk Tanja Schmitt \u2190",
"startOffset": 170,
"endOffset": 192,
"type": "ORG"
}
],
"2": [
{
"value": "Rue Jean Baffier",
"startOffset": 214,
"endOffset": 230,
"type": "CBI_author"
},
{
"value": "7232",
"startOffset": 155,
"endOffset": 159,
"type": "POSTAL"
},
{
"value": "CX",
"startOffset": 160,
"endOffset": 162,
"type": "COUNTRY"
},
{
"value": "Warnsveld",
"startOffset": 163,
"endOffset": 172,
"type": "CITY"
},
{
"value": "Netherlands",
"startOffset": 174,
"endOffset": 185,
"type": "COUNTRY"
},
{
"value": "NL",
"startOffset": 187,
"endOffset": 189,
"type": "COUNTRY"
},
{
"value": "Institut Industries",
"startOffset": 190,
"endOffset": 209,
"type": "ORG"
},
{
"value": "33",
"startOffset": 211,
"endOffset": 213,
"type": "CARDINAL"
},
{
"value": "Rue Jean Baffier",
"startOffset": 214,
"endOffset": 230,
"type": "STREET"
},
{
"value": "18000",
"startOffset": 232,
"endOffset": 237,
"type": "CARDINAL"
},
{
"value": "Bourges",
"startOffset": 238,
"endOffset": 245,
"type": "CITY"
},
{
"value": "France",
"startOffset": 247,
"endOffset": 253,
"type": "COUNTRY"
},
{
"value": "18300",
"startOffset": 282,
"endOffset": 287,
"type": "CARDINAL"
},
{
"value": "Saint-Satur",
"startOffset": 288,
"endOffset": 299,
"type": "CITY"
},
{
"value": "France",
"startOffset": 301,
"endOffset": 307,
"type": "COUNTRY"
},
{
"value": "Lesdo Industries",
"startOffset": 312,
"endOffset": 328,
"type": "ORG"
},
{
"value": "Ch\u00e4ppelistr\u00e4ssli",
"startOffset": 330,
"endOffset": 346,
"type": "ORG"
},
{
"value": "6078",
"startOffset": 348,
"endOffset": 352,
"type": "POSTAL"
},
{
"value": "Lungern",
"startOffset": 353,
"endOffset": 360,
"type": "STREET"
},
{
"value": "Switzerland",
"startOffset": 362,
"endOffset": 373,
"type": "COUNTRY"
},
{
"value": "Shlissel'burgskaya Ulitsa",
"startOffset": 374,
"endOffset": 399,
"type": "ORG"
},
{
"value": "Nizhny Novgorod Oblast",
"startOffset": 401,
"endOffset": 423,
"type": "STREET"
},
{
"value": "Russia",
"startOffset": 425,
"endOffset": 431,
"type": "CITY"
},
{
"value": "603034",
"startOffset": 433,
"endOffset": 439,
"type": "POSTAL"
},
{
"value": "RU",
"startOffset": 441,
"endOffset": 443,
"type": "COUNTRY"
},
{
"value": "Karl Johans Gate",
"startOffset": 444,
"endOffset": 460,
"type": "STREET"
},
{
"value": "11",
"startOffset": 461,
"endOffset": 463,
"type": "CARDINAL"
},
{
"value": "0154",
"startOffset": 465,
"endOffset": 469,
"type": "POSTAL"
},
{
"value": "Oslo",
"startOffset": 470,
"endOffset": 474,
"type": "CITY"
},
{
"value": "Norway",
"startOffset": 476,
"endOffset": 482,
"type": "COUNTRY"
}
],
"3": [
{
"value": "Expand",
"startOffset": 67,
"endOffset": 73,
"type": "STATE"
},
{
"value": "Hint Clarissa",
"startOffset": 77,
"endOffset": 90,
"type": "ORG"
},
{
"value": "Dict",
"startOffset": 114,
"endOffset": 118,
"type": "ORG"
},
{
"value": "Authors-Dict",
"startOffset": 171,
"endOffset": 183,
"type": "ORG"
}
],
"4": [
{
"value": "Michael N.",
"startOffset": 149,
"endOffset": 159,
"type": "CBI_author"
},
{
"value": "Funnarie B.",
"startOffset": 244,
"endOffset": 255,
"type": "CBI_author"
},
{
"value": "Weyland Industries",
"startOffset": 218,
"endOffset": 236,
"type": "ORG"
},
{
"value": "Tyrell Corporation",
"startOffset": 406,
"endOffset": 424,
"type": "ORG"
}
],
"6": [
{
"value": "Melanie",
"startOffset": 292,
"endOffset": 299,
"type": "CBI_author"
}
],
"7": [
{
"value": "Stark Industries",
"startOffset": 184,
"endOffset": 200,
"type": "ORG"
}
],
"8": [
{
"value": "Omni Consumer Products Do",
"startOffset": 197,
"endOffset": 222,
"type": "ORG"
}
],
"9": [
{
"value": "Omni Consumer Products Do",
"startOffset": 122,
"endOffset": 147,
"type": "ORG"
}
],
"10": [
{
"value": "Asya Lyon",
"startOffset": 253,
"endOffset": 262,
"type": "CBI_author"
},
{
"value": "Carina Madsen",
"startOffset": 264,
"endOffset": 277,
"type": "CBI_author"
},
{
"value": "Alexandra H\u00e4usler",
"startOffset": 279,
"endOffset": 296,
"type": "CBI_author"
},
{
"value": "Hanke Mendel",
"startOffset": 298,
"endOffset": 310,
"type": "CBI_author"
},
{
"value": "Kwok, Jun K.",
"startOffset": 444,
"endOffset": 456,
"type": "CBI_author"
},
{
"value": "Tu Wong",
"startOffset": 458,
"endOffset": 465,
"type": "CBI_author"
},
{
"value": "Qiang Suen",
"startOffset": 467,
"endOffset": 477,
"type": "CBI_author"
},
{
"value": "Zhou Mah",
"startOffset": 479,
"endOffset": 487,
"type": "CBI_author"
},
{
"value": "Lei W. Huang",
"startOffset": 499,
"endOffset": 511,
"type": "CBI_author"
},
{
"value": "Ru X.",
"startOffset": 513,
"endOffset": 518,
"type": "CBI_author"
},
{
"value": "Oxford University Press",
"startOffset": 166,
"endOffset": 189,
"type": "ORG"
},
{
"value": "Iakovos Geiger",
"startOffset": 222,
"endOffset": 236,
"type": "ORG"
},
{
"value": "Julian Ritter",
"startOffset": 238,
"endOffset": 251,
"type": "CITY"
},
{
"value": "Asya Lyon",
"startOffset": 253,
"endOffset": 262,
"type": "ORG"
},
{
"value": "Carina Madsen",
"startOffset": 264,
"endOffset": 277,
"type": "CITY"
},
{
"value": "Alexandra H\u00e4usler",
"startOffset": 279,
"endOffset": 296,
"type": "ORG"
},
{
"value": "Hanke Mendel",
"startOffset": 298,
"endOffset": 310,
"type": "ORG"
},
{
"value": "Ranya",
"startOffset": 312,
"endOffset": 317,
"type": "COUNTRY"
},
{
"value": "Eikenboom",
"startOffset": 318,
"endOffset": 327,
"type": "ORG"
},
{
"value": "Min Kwok",
"startOffset": 440,
"endOffset": 448,
"type": "ORG"
},
{
"value": "Qiang Suen",
"startOffset": 467,
"endOffset": 477,
"type": "CITY"
},
{
"value": "Zhou Mah",
"startOffset": 479,
"endOffset": 487,
"type": "ORG"
},
{
"value": "Ning Liu",
"startOffset": 489,
"endOffset": 497,
"type": "STREET"
},
{
"value": "Lei W. Huang, Ru X. Wu",
"startOffset": 499,
"endOffset": 521,
"type": "ORG"
}
],
"11": [
{
"value": "Nurullah \u00d6zg\u00fcr",
"startOffset": 210,
"endOffset": 224,
"type": "CBI_author"
},
{
"value": "Reyhan B.",
"startOffset": 228,
"endOffset": 237,
"type": "CBI_author"
},
{
"value": "Alfred Xinyi Y.",
"startOffset": 250,
"endOffset": 265,
"type": "CBI_author"
},
{
"value": "Redacted",
"startOffset": 12,
"endOffset": 20,
"type": "ORG"
},
{
"value": "Aomachi",
"startOffset": 154,
"endOffset": 161,
"type": "ORG"
},
{
"value": "Nomi",
"startOffset": 163,
"endOffset": 167,
"type": "ORG"
},
{
"value": "Ishikawa",
"startOffset": 169,
"endOffset": 177,
"type": "CITY"
},
{
"value": "923-1101",
"startOffset": 178,
"endOffset": 186,
"type": "CARDINAL"
},
{
"value": "Japan",
"startOffset": 188,
"endOffset": 193,
"type": "COUNTRY"
},
{
"value": "JP",
"startOffset": 195,
"endOffset": 197,
"type": "COUNTRY"
},
{
"value": "Sude Halide Nurullah \u00d6zg\u00fcr U. Reyhan B. Rahim C. J. Alfred Xinyi Y. Tao Clara Siegfried",
"startOffset": 198,
"endOffset": 285,
"type": "ORG"
},
{
"value": "Dict",
"startOffset": 301,
"endOffset": 305,
"type": "ORG"
}
],
"12": [
{
"value": "Redact Emails",
"startOffset": 12,
"endOffset": 25,
"type": "ORG"
}
],
"13": [
{
"value": "Central Research Industry",
"startOffset": 462,
"endOffset": 487,
"type": "ORG"
},
{
"value": "Maximiliam Schmitt",
"startOffset": 718,
"endOffset": 736,
"type": "ORG"
},
{
"value": "European Central Institute",
"startOffset": 915,
"endOffset": 941,
"type": "ORG"
},
{
"value": "Emilia Lockhart Alternative",
"startOffset": 963,
"endOffset": 990,
"type": "ORG"
},
{
"value": "Cyberdyne Systems Tower",
"startOffset": 1000,
"endOffset": 1023,
"type": "ORG"
},
{
"value": "121a",
"startOffset": 1032,
"endOffset": 1036,
"type": "CARDINAL"
},
{
"value": "Hong Kong",
"startOffset": 1037,
"endOffset": 1046,
"type": "COUNTRY"
},
{
"value": "BT",
"startOffset": 1048,
"endOffset": 1050,
"type": "COUNTRY"
}
],
"14": [
{
"value": "Soylent Corporation",
"startOffset": 221,
"endOffset": 240,
"type": "ORG"
},
{
"value": "Riddley",
"startOffset": 256,
"endOffset": 263,
"type": "ORG"
},
{
"value": "359-21",
"startOffset": 279,
"endOffset": 285,
"type": "CARDINAL"
},
{
"value": "Huam-dong",
"startOffset": 286,
"endOffset": 295,
"type": "STATE"
},
{
"value": "Yongsan-gu",
"startOffset": 296,
"endOffset": 306,
"type": "CITY"
},
{
"value": "Seoul",
"startOffset": 307,
"endOffset": 312,
"type": "CITY"
},
{
"value": "South Korea Phone",
"startOffset": 314,
"endOffset": 331,
"type": "COUNTRY"
},
{
"value": "Central Research Industry",
"startOffset": 790,
"endOffset": 815,
"type": "ORG"
},
{
"value": "Maximiliam Schmitt",
"startOffset": 1046,
"endOffset": 1064,
"type": "ORG"
},
{
"value": "European Central Institute",
"startOffset": 1243,
"endOffset": 1269,
"type": "ORG"
},
{
"value": "Emilia Lockhart Alternative",
"startOffset": 1291,
"endOffset": 1318,
"type": "ORG"
},
{
"value": "Cyberdyne Systems Tower",
"startOffset": 1328,
"endOffset": 1351,
"type": "ORG"
},
{
"value": "121a",
"startOffset": 1360,
"endOffset": 1364,
"type": "CARDINAL"
},
{
"value": "Hong Kong",
"startOffset": 1365,
"endOffset": 1374,
"type": "COUNTRY"
},
{
"value": "BT",
"startOffset": 1376,
"endOffset": 1378,
"type": "COUNTRY"
}
],
"15": [
{
"value": "Umbrella Corporation",
"startOffset": 208,
"endOffset": 228,
"type": "ORG"
},
{
"value": "Jill",
"startOffset": 238,
"endOffset": 242,
"type": "ORG"
},
{
"value": "359-21",
"startOffset": 262,
"endOffset": 268,
"type": "CARDINAL"
},
{
"value": "Huam-dong",
"startOffset": 269,
"endOffset": 278,
"type": "STATE"
},
{
"value": "Yongsan-gu",
"startOffset": 279,
"endOffset": 289,
"type": "CITY"
},
{
"value": "Seoul",
"startOffset": 290,
"endOffset": 295,
"type": "CITY"
},
{
"value": "South Korea Phone",
"startOffset": 297,
"endOffset": 314,
"type": "COUNTRY"
}
],
"18": [
{
"value": "Umbrella Corporation",
"startOffset": 209,
"endOffset": 229,
"type": "ORG"
}
],
"20": [
{
"value": "Purity Hint",
"startOffset": 9,
"endOffset": 20,
"type": "ORG"
},
{
"value": "Hint",
"startOffset": 35,
"endOffset": 39,
"type": "ORG"
},
{
"value": "Hint Purity",
"startOffset": 170,
"endOffset": 181,
"type": "ORG"
}
],
"21": [
{
"value": "Ignore",
"startOffset": 9,
"endOffset": 15,
"type": "STREET"
}
],
"22": [
{
"value": "Redact Signatures Redact",
"startOffset": 12,
"endOffset": 36,
"type": "ORG"
},
{
"value": "Dilara Sonnenschein Signed",
"startOffset": 166,
"endOffset": 192,
"type": "ORG"
},
{
"value": "Tobias M\u00fcller",
"startOffset": 197,
"endOffset": 210,
"type": "ORG"
}
],
"23": [
{
"value": "Redact Logo Redact",
"startOffset": 9,
"endOffset": 27,
"type": "ORG"
}
]
}
}