diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java index 9ca514e8..ffa47a56 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java @@ -1,7 +1,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.adapter; import java.util.Collection; +import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -9,48 +11,32 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @Service -@RequiredArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class NerEntitiesAdapter { - RedactionStorageService redactionStorageService; - RedactionServiceSettings redactionServiceSettings; - // imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition - private static final Set CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE"); - private static final Set CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY"); - private static final int MAX_DISTANCE_BETWEEN_PARTS = 20; - private static final int MIN_PART_MATCHES = 3; - private static final boolean ALLOW_DUPLICATES = false; + static Set CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE"); + static Set CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY"); + + static int MAX_DISTANCE_BETWEEN_PARTS = 20; + static int MIN_PART_MATCHES = 3; + static boolean ALLOW_DUPLICATES; - public List getNerEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) { + public List validateAndCombine(NerEntities nerEntities, DocumentGraph documentGraph) { - NerEntities nerEntities; - if (redactionServiceSettings.isNerServiceEnabled()) { - nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - } else { - nerEntities = new NerEntities(); - } - - addSectionStartOffsetToEachNerEntity(getStringStartOffsetsForMainSections(documentGraph), nerEntities); - - List entityRecognitionEntities = validateForCorrectOffsets(documentGraph, nerEntities); + List entityRecognitionEntities = getValidatedEntities(nerEntities, documentGraph).toList(); List cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList(); List cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock()); @@ -59,26 +45,65 @@ public class NerEntitiesAdapter { } + protected Stream getValidatedEntities(NerEntities nerEntities, DocumentGraph documentGraph) { + + return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntities) // + .filter(nerEntity -> nerEntityOffsetMatchesDocumentGraph(nerEntity, documentGraph.buildTextBlock())); + } + + private List combineToCBIAddressEntities(List entityRecognitionEntities, TextBlock textBlock) { - Set entitiesOfTypeCBIAddressParts = entityRecognitionEntities.stream() + List sortedEntities = entityRecognitionEntities.stream() .filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType())) - .collect(Collectors.toSet()); + .sorted(Comparator.comparingInt(EntityRecognitionEntity::getStartOffset)) + .toList(); - return entitiesOfTypeCBIAddressParts.stream() - .filter(entity -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(entity.getType())) - .map(entity -> findFollowingEntitiesOfTypeCBIAddressPart(entity, entitiesOfTypeCBIAddressParts)) - .filter(entities -> entities.size() < MIN_PART_MATCHES) - .map(this::toContainingBoundary) - .distinct() + if (sortedEntities.isEmpty()) { + return sortedEntities; + } + + List> entityClusters = new LinkedList<>(); + + List startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(e.getType())).toList(); + for (EntityRecognitionEntity startEntity : startEntitiesOfEssentialType) { + List currentCluster = new LinkedList<>(); + entityClusters.add(currentCluster); + int lastEndOffset = startEntity.getStartOffset(); + + for (EntityRecognitionEntity entity : sortedEntities) { + if (entity.getStartOffset() < lastEndOffset) { + continue; + } + if (disctanceIsLargerThanMaxDistance(lastEndOffset, entity) || isDuplicate(currentCluster, entity)) { + currentCluster = new LinkedList<>(); + entityClusters.add(currentCluster); + currentCluster.add(entity); + lastEndOffset = entity.getEndOffset(); + } else { + currentCluster.add(entity); + lastEndOffset = entity.getEndOffset(); + } + } + } + + List boundaries = entityClusters.stream().filter(cluster -> cluster.size() >= MIN_PART_MATCHES).map(this::toContainingBoundary).distinct().toList(); + + return boundaries.stream() .map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address")) .toList(); } - private List validateForCorrectOffsets(DocumentGraph documentGraph, NerEntities nerEntities) { + private static boolean isDuplicate(List currentCluster, EntityRecognitionEntity entity) { - return nerEntities.getData().values().stream().flatMap(Collection::stream).filter(nerEntity -> nerEntityOffsetMatches(nerEntity, documentGraph.buildTextBlock())).toList(); + return ALLOW_DUPLICATES || currentCluster.stream().anyMatch(e -> e.getType().equals(entity.getType())); + } + + + private static boolean disctanceIsLargerThanMaxDistance(int lastEndOffset, EntityRecognitionEntity entity) { + + return (entity.getStartOffset() - lastEndOffset) > MAX_DISTANCE_BETWEEN_PARTS; } @@ -122,19 +147,20 @@ public class NerEntitiesAdapter { } - private boolean nerEntityOffsetMatches(EntityRecognitionEntity nerEntity, TextBlock textBlock) { + private boolean nerEntityOffsetMatchesDocumentGraph(EntityRecognitionEntity nerEntity, TextBlock textBlock) { return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset())); } - private static void addSectionStartOffsetToEachNerEntity(List stringOffsetsForMainSections, NerEntities nerEntities) { + private static Stream addOffsetsAndFlatten(List stringOffsetsForMainSections, NerEntities nerEntities) { nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> { int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key); entityRecognitionEntity.setStartOffset(newStartOffset); entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); })); + return nerEntities.getData().values().stream().flatMap(Collection::stream); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index 6aa9e7de..dab4c916 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze; import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -166,7 +167,12 @@ public class AnalyzeService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode)); - List nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph); + List nerEntities; + if (redactionServiceSettings.isNerServiceEnabled()) { + nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph); + } else { + nerEntities = Collections.emptyList(); + } KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); @@ -198,7 +204,13 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - List nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph); + List nerEntities; + if (redactionServiceSettings.isNerServiceEnabled()) { + nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph); + } else { + nerEntities = Collections.emptyList(); + } + dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphIntegrationTest.java index e4fe6f75..c8bf69ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphIntegrationTest.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import java.io.InputStream; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,12 +62,15 @@ public class BuildDocumentGraphIntegrationTest extends AbstractRedactionIntegrat @SneakyThrows protected DocumentGraph buildGraph(String filename) { - if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) { - prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json"); - } else { - uploadFileToStorage(filename + ".pdf"); + if (!filename.endsWith(".pdf")) { + filename = filename + ".pdf"; } - ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) { + prepareStorage(filename, "files/cv_service_empty_response.json", filename.replace(".pdf", ".IMAGE_INFO.json")); + } else { + uploadFileToStorage(filename); + } + ClassPathResource fileResource = new ClassPathResource(filename); try (InputStream inputStream = fileResource.getInputStream()) { Map> pdfImages = imageServiceResponseAdapter.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID); @@ -75,4 +79,23 @@ public class BuildDocumentGraphIntegrationTest extends AbstractRedactionIntegrat } } + + @SneakyThrows + protected DocumentGraph buildGraphNoImages(String filename) { + + if (!filename.endsWith(".pdf")) { + filename = filename + ".pdf"; + } + + uploadFileToStorage(filename); + + ClassPathResource fileResource = new ClassPathResource(filename); + + try (InputStream inputStream = fileResource.getInputStream()) { + Map> pdfImages = new HashMap<>(); + Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages); + return DocumentGraphFactory.buildDocumentGraph(classifiedDoc); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionIntegrationTest.java index ad5276be..95582bb5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionIntegrationTest.java @@ -33,7 +33,7 @@ public class DocumentGraphEntityInsertionIntegrationTest extends BuildDocumentGr @Test public void assertCollectAllEntitiesWorks() { - DocumentGraph documentGraph = buildGraph("files/new/crafted document"); + DocumentGraph documentGraph = buildGraph("files/new/crafted document.pdf"); createAndInsertEntity(documentGraph, "Clarissa"); createAndInsertEntity(documentGraph, "Lastname"); createAndInsertEntity(documentGraph, "David Ksenia"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java new file mode 100644 index 00000000..9640c37f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -0,0 +1,119 @@ +package com.iqser.red.service.redaction.v1.server.redaction.adapter; + +import static org.wildfly.common.Assert.assertFalse; +import static org.wildfly.common.Assert.assertTrue; + +import java.awt.Color; +import java.awt.geom.Rectangle2D; +import java.io.File; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw; + +import lombok.SneakyThrows; + +class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest { + + @Autowired + private NerEntitiesAdapter nerEntitiesAdapter; + @Autowired + private EntityCreationService entityCreationService; + + + @SneakyThrows + private static NerEntities parseNerEntities(String filePath) { + + ClassPathResource resource = new ClassPathResource(filePath); + return ObjectMapperFactory.create().readValue(resource.getInputStream(), NerEntities.class); + } + + + @Test + @SneakyThrows + public void testGetNerEntities() { + + String filePath = "files/new/crafted document.pdf"; + String nerEntitiesFilePath = "ner_entities/crafted document.NER_ENTITIES.json"; + DocumentGraph documentGraph = buildGraphNoImages(filePath); + List entityRecognitionEntities = nerEntitiesAdapter.validateAndCombine(parseNerEntities(nerEntitiesFilePath), documentGraph); + assertFalse(entityRecognitionEntities.isEmpty()); + assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.getStartOffset() < entity.getEndOffset())); + + ClassPathResource resource = new ClassPathResource(filePath); + PDDocument pdDocument = PDDocument.load(resource.getInputStream()); + Stream unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph) + .filter(e -> !e.getType().equals("CBI_author")); + List redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts) + .map(e -> entityCreationService.byBoundary(new Boundary(e.getStartOffset(), e.getEndOffset()), e.getType(), EntityType.ENTITY, documentGraph)) + .toList(); + redactionEntities.stream() + .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) + .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber, + getPositionsFromEntityOfType("CBI_author", entities), + getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), + getPositionsFromEntityOfType("CBI_address", entities), + pdDocument)); + + File outputFile = new File("/tmp/nerEntities.pdf"); + pdDocument.save(outputFile); + pdDocument.close(); + + } + + + private List getPositionsFromEntities(Stream entities) { + + return entities.map(RedactionEntity::getRedactionPositionsPerPage) + .flatMap(Collection::stream) + .map(RedactionPosition::getRectanglePerLine) + .flatMap(Collection::stream) + .toList(); + } + + + private List getPositionsFromEntityOfType(String type, List entities) { + + return getPositionsFromEntities(entities.stream().filter(e -> e.getType().equals(type))); + + } + + + private List getPositionsFromEntityNotOfType(List types, List entities) { + + return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.getType().equals(type)))); + + } + + + @SneakyThrows + private static void drawNerEntitiesAsPartsAndCombined(int pageNumber, + List cbiAuthorRects, + List addressPartsRects, + List cbiAddressRects, + PDDocument pdDocument) { + + //PdfDraw.drawRectangle2DList(pdDocument, pageNumber, cbiAuthorRects, PdfDraw.Options.builder().stroke(true).strokeColor(Color.blue).build()); + PdfDraw.drawRectangle2DList(pdDocument, pageNumber, addressPartsRects, PdfDraw.Options.builder().stroke(true).strokeColor(Color.MAGENTA).build()); + PdfDraw.drawRectangle2DList(pdDocument, pageNumber, cbiAddressRects, PdfDraw.Options.builder().stroke(true).strokeColor(Color.green).build()); + + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/ner_entities/crafted document.NER_ENTITIES.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/ner_entities/crafted document.NER_ENTITIES.json new file mode 100644 index 00000000..dcc192dd --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/ner_entities/crafted document.NER_ENTITIES.json @@ -0,0 +1,792 @@ +{ + "dossierId": "ef6d4541-4fea-4451-932a-55d167ab30dc", + "fileId": "44a0ab4b930ac6bd89dbbfa7c23f9c98", + "targetFileExtension": "SIMPLIFIED_TEXT.json.gz", + "responseFileExtension": "NER_ENTITIES.json.gz", + "data": { + "0": [ + { + "value": "Lastname, J.", + "startOffset": 54, + "endOffset": 66, + "type": "CBI_author" + }, + { + "value": "Doe, M.", + "startOffset": 67, + "endOffset": 74, + "type": "CBI_author" + }, + { + "value": "Mustermann Lastname M.", + "startOffset": 75, + "endOffset": 97, + "type": "CBI_author" + }, + { + "value": "Doe J. Mustermann M.", + "startOffset": 99, + "endOffset": 119, + "type": "CBI_author" + } + ], + "1": [ + { + "value": "Eikenboom Charalampos", + "startOffset": 148, + "endOffset": 169, + "type": "ORG" + }, + { + "value": "Schenk Tanja Schmitt \u2190", + "startOffset": 170, + "endOffset": 192, + "type": "ORG" + } + ], + "2": [ + { + "value": "Rue Jean Baffier", + "startOffset": 214, + "endOffset": 230, + "type": "CBI_author" + }, + { + "value": "7232", + "startOffset": 155, + "endOffset": 159, + "type": "POSTAL" + }, + { + "value": "CX", + "startOffset": 160, + "endOffset": 162, + "type": "COUNTRY" + }, + { + "value": "Warnsveld", + "startOffset": 163, + "endOffset": 172, + "type": "CITY" + }, + { + "value": "Netherlands", + "startOffset": 174, + "endOffset": 185, + "type": "COUNTRY" + }, + { + "value": "NL", + "startOffset": 187, + "endOffset": 189, + "type": "COUNTRY" + }, + { + "value": "Institut Industries", + "startOffset": 190, + "endOffset": 209, + "type": "ORG" + }, + { + "value": "33", + "startOffset": 211, + "endOffset": 213, + "type": "CARDINAL" + }, + { + "value": "Rue Jean Baffier", + "startOffset": 214, + "endOffset": 230, + "type": "STREET" + }, + { + "value": "18000", + "startOffset": 232, + "endOffset": 237, + "type": "CARDINAL" + }, + { + "value": "Bourges", + "startOffset": 238, + "endOffset": 245, + "type": "CITY" + }, + { + "value": "France", + "startOffset": 247, + "endOffset": 253, + "type": "COUNTRY" + }, + { + "value": "18300", + "startOffset": 282, + "endOffset": 287, + "type": "CARDINAL" + }, + { + "value": "Saint-Satur", + "startOffset": 288, + "endOffset": 299, + "type": "CITY" + }, + { + "value": "France", + "startOffset": 301, + "endOffset": 307, + "type": "COUNTRY" + }, + { + "value": "Lesdo Industries", + "startOffset": 312, + "endOffset": 328, + "type": "ORG" + }, + { + "value": "Ch\u00e4ppelistr\u00e4ssli", + "startOffset": 330, + "endOffset": 346, + "type": "ORG" + }, + { + "value": "6078", + "startOffset": 348, + "endOffset": 352, + "type": "POSTAL" + }, + { + "value": "Lungern", + "startOffset": 353, + "endOffset": 360, + "type": "STREET" + }, + { + "value": "Switzerland", + "startOffset": 362, + "endOffset": 373, + "type": "COUNTRY" + }, + { + "value": "Shlissel'burgskaya Ulitsa", + "startOffset": 374, + "endOffset": 399, + "type": "ORG" + }, + { + "value": "Nizhny Novgorod Oblast", + "startOffset": 401, + "endOffset": 423, + "type": "STREET" + }, + { + "value": "Russia", + "startOffset": 425, + "endOffset": 431, + "type": "CITY" + }, + { + "value": "603034", + "startOffset": 433, + "endOffset": 439, + "type": "POSTAL" + }, + { + "value": "RU", + "startOffset": 441, + "endOffset": 443, + "type": "COUNTRY" + }, + { + "value": "Karl Johans Gate", + "startOffset": 444, + "endOffset": 460, + "type": "STREET" + }, + { + "value": "11", + "startOffset": 461, + "endOffset": 463, + "type": "CARDINAL" + }, + { + "value": "0154", + "startOffset": 465, + "endOffset": 469, + "type": "POSTAL" + }, + { + "value": "Oslo", + "startOffset": 470, + "endOffset": 474, + "type": "CITY" + }, + { + "value": "Norway", + "startOffset": 476, + "endOffset": 482, + "type": "COUNTRY" + } + ], + "3": [ + { + "value": "Expand", + "startOffset": 67, + "endOffset": 73, + "type": "STATE" + }, + { + "value": "Hint Clarissa", + "startOffset": 77, + "endOffset": 90, + "type": "ORG" + }, + { + "value": "Dict", + "startOffset": 114, + "endOffset": 118, + "type": "ORG" + }, + { + "value": "Authors-Dict", + "startOffset": 171, + "endOffset": 183, + "type": "ORG" + } + ], + "4": [ + { + "value": "Michael N.", + "startOffset": 149, + "endOffset": 159, + "type": "CBI_author" + }, + { + "value": "Funnarie B.", + "startOffset": 244, + "endOffset": 255, + "type": "CBI_author" + }, + { + "value": "Weyland Industries", + "startOffset": 218, + "endOffset": 236, + "type": "ORG" + }, + { + "value": "Tyrell Corporation", + "startOffset": 406, + "endOffset": 424, + "type": "ORG" + } + ], + "6": [ + { + "value": "Melanie", + "startOffset": 292, + "endOffset": 299, + "type": "CBI_author" + } + ], + "7": [ + { + "value": "Stark Industries", + "startOffset": 184, + "endOffset": 200, + "type": "ORG" + } + ], + "8": [ + { + "value": "Omni Consumer Products Do", + "startOffset": 197, + "endOffset": 222, + "type": "ORG" + } + ], + "9": [ + { + "value": "Omni Consumer Products Do", + "startOffset": 122, + "endOffset": 147, + "type": "ORG" + } + ], + "10": [ + { + "value": "Asya Lyon", + "startOffset": 253, + "endOffset": 262, + "type": "CBI_author" + }, + { + "value": "Carina Madsen", + "startOffset": 264, + "endOffset": 277, + "type": "CBI_author" + }, + { + "value": "Alexandra H\u00e4usler", + "startOffset": 279, + "endOffset": 296, + "type": "CBI_author" + }, + { + "value": "Hanke Mendel", + "startOffset": 298, + "endOffset": 310, + "type": "CBI_author" + }, + { + "value": "Kwok, Jun K.", + "startOffset": 444, + "endOffset": 456, + "type": "CBI_author" + }, + { + "value": "Tu Wong", + "startOffset": 458, + "endOffset": 465, + "type": "CBI_author" + }, + { + "value": "Qiang Suen", + "startOffset": 467, + "endOffset": 477, + "type": "CBI_author" + }, + { + "value": "Zhou Mah", + "startOffset": 479, + "endOffset": 487, + "type": "CBI_author" + }, + { + "value": "Lei W. Huang", + "startOffset": 499, + "endOffset": 511, + "type": "CBI_author" + }, + { + "value": "Ru X.", + "startOffset": 513, + "endOffset": 518, + "type": "CBI_author" + }, + { + "value": "Oxford University Press", + "startOffset": 166, + "endOffset": 189, + "type": "ORG" + }, + { + "value": "Iakovos Geiger", + "startOffset": 222, + "endOffset": 236, + "type": "ORG" + }, + { + "value": "Julian Ritter", + "startOffset": 238, + "endOffset": 251, + "type": "CITY" + }, + { + "value": "Asya Lyon", + "startOffset": 253, + "endOffset": 262, + "type": "ORG" + }, + { + "value": "Carina Madsen", + "startOffset": 264, + "endOffset": 277, + "type": "CITY" + }, + { + "value": "Alexandra H\u00e4usler", + "startOffset": 279, + "endOffset": 296, + "type": "ORG" + }, + { + "value": "Hanke Mendel", + "startOffset": 298, + "endOffset": 310, + "type": "ORG" + }, + { + "value": "Ranya", + "startOffset": 312, + "endOffset": 317, + "type": "COUNTRY" + }, + { + "value": "Eikenboom", + "startOffset": 318, + "endOffset": 327, + "type": "ORG" + }, + { + "value": "Min Kwok", + "startOffset": 440, + "endOffset": 448, + "type": "ORG" + }, + { + "value": "Qiang Suen", + "startOffset": 467, + "endOffset": 477, + "type": "CITY" + }, + { + "value": "Zhou Mah", + "startOffset": 479, + "endOffset": 487, + "type": "ORG" + }, + { + "value": "Ning Liu", + "startOffset": 489, + "endOffset": 497, + "type": "STREET" + }, + { + "value": "Lei W. Huang, Ru X. Wu", + "startOffset": 499, + "endOffset": 521, + "type": "ORG" + } + ], + "11": [ + { + "value": "Nurullah \u00d6zg\u00fcr", + "startOffset": 210, + "endOffset": 224, + "type": "CBI_author" + }, + { + "value": "Reyhan B.", + "startOffset": 228, + "endOffset": 237, + "type": "CBI_author" + }, + { + "value": "Alfred Xinyi Y.", + "startOffset": 250, + "endOffset": 265, + "type": "CBI_author" + }, + { + "value": "Redacted", + "startOffset": 12, + "endOffset": 20, + "type": "ORG" + }, + { + "value": "Aomachi", + "startOffset": 154, + "endOffset": 161, + "type": "ORG" + }, + { + "value": "Nomi", + "startOffset": 163, + "endOffset": 167, + "type": "ORG" + }, + { + "value": "Ishikawa", + "startOffset": 169, + "endOffset": 177, + "type": "CITY" + }, + { + "value": "923-1101", + "startOffset": 178, + "endOffset": 186, + "type": "CARDINAL" + }, + { + "value": "Japan", + "startOffset": 188, + "endOffset": 193, + "type": "COUNTRY" + }, + { + "value": "JP", + "startOffset": 195, + "endOffset": 197, + "type": "COUNTRY" + }, + { + "value": "Sude Halide Nurullah \u00d6zg\u00fcr U. Reyhan B. Rahim C. J. Alfred Xinyi Y. Tao Clara Siegfried", + "startOffset": 198, + "endOffset": 285, + "type": "ORG" + }, + { + "value": "Dict", + "startOffset": 301, + "endOffset": 305, + "type": "ORG" + } + ], + "12": [ + { + "value": "Redact Emails", + "startOffset": 12, + "endOffset": 25, + "type": "ORG" + } + ], + "13": [ + { + "value": "Central Research Industry", + "startOffset": 462, + "endOffset": 487, + "type": "ORG" + }, + { + "value": "Maximiliam Schmitt", + "startOffset": 718, + "endOffset": 736, + "type": "ORG" + }, + { + "value": "European Central Institute", + "startOffset": 915, + "endOffset": 941, + "type": "ORG" + }, + { + "value": "Emilia Lockhart Alternative", + "startOffset": 963, + "endOffset": 990, + "type": "ORG" + }, + { + "value": "Cyberdyne Systems Tower", + "startOffset": 1000, + "endOffset": 1023, + "type": "ORG" + }, + { + "value": "121a", + "startOffset": 1032, + "endOffset": 1036, + "type": "CARDINAL" + }, + { + "value": "Hong Kong", + "startOffset": 1037, + "endOffset": 1046, + "type": "COUNTRY" + }, + { + "value": "BT", + "startOffset": 1048, + "endOffset": 1050, + "type": "COUNTRY" + } + ], + "14": [ + { + "value": "Soylent Corporation", + "startOffset": 221, + "endOffset": 240, + "type": "ORG" + }, + { + "value": "Riddley", + "startOffset": 256, + "endOffset": 263, + "type": "ORG" + }, + { + "value": "359-21", + "startOffset": 279, + "endOffset": 285, + "type": "CARDINAL" + }, + { + "value": "Huam-dong", + "startOffset": 286, + "endOffset": 295, + "type": "STATE" + }, + { + "value": "Yongsan-gu", + "startOffset": 296, + "endOffset": 306, + "type": "CITY" + }, + { + "value": "Seoul", + "startOffset": 307, + "endOffset": 312, + "type": "CITY" + }, + { + "value": "South Korea Phone", + "startOffset": 314, + "endOffset": 331, + "type": "COUNTRY" + }, + { + "value": "Central Research Industry", + "startOffset": 790, + "endOffset": 815, + "type": "ORG" + }, + { + "value": "Maximiliam Schmitt", + "startOffset": 1046, + "endOffset": 1064, + "type": "ORG" + }, + { + "value": "European Central Institute", + "startOffset": 1243, + "endOffset": 1269, + "type": "ORG" + }, + { + "value": "Emilia Lockhart Alternative", + "startOffset": 1291, + "endOffset": 1318, + "type": "ORG" + }, + { + "value": "Cyberdyne Systems Tower", + "startOffset": 1328, + "endOffset": 1351, + "type": "ORG" + }, + { + "value": "121a", + "startOffset": 1360, + "endOffset": 1364, + "type": "CARDINAL" + }, + { + "value": "Hong Kong", + "startOffset": 1365, + "endOffset": 1374, + "type": "COUNTRY" + }, + { + "value": "BT", + "startOffset": 1376, + "endOffset": 1378, + "type": "COUNTRY" + } + ], + "15": [ + { + "value": "Umbrella Corporation", + "startOffset": 208, + "endOffset": 228, + "type": "ORG" + }, + { + "value": "Jill", + "startOffset": 238, + "endOffset": 242, + "type": "ORG" + }, + { + "value": "359-21", + "startOffset": 262, + "endOffset": 268, + "type": "CARDINAL" + }, + { + "value": "Huam-dong", + "startOffset": 269, + "endOffset": 278, + "type": "STATE" + }, + { + "value": "Yongsan-gu", + "startOffset": 279, + "endOffset": 289, + "type": "CITY" + }, + { + "value": "Seoul", + "startOffset": 290, + "endOffset": 295, + "type": "CITY" + }, + { + "value": "South Korea Phone", + "startOffset": 297, + "endOffset": 314, + "type": "COUNTRY" + } + ], + "18": [ + { + "value": "Umbrella Corporation", + "startOffset": 209, + "endOffset": 229, + "type": "ORG" + } + ], + "20": [ + { + "value": "Purity Hint", + "startOffset": 9, + "endOffset": 20, + "type": "ORG" + }, + { + "value": "Hint", + "startOffset": 35, + "endOffset": 39, + "type": "ORG" + }, + { + "value": "Hint Purity", + "startOffset": 170, + "endOffset": 181, + "type": "ORG" + } + ], + "21": [ + { + "value": "Ignore", + "startOffset": 9, + "endOffset": 15, + "type": "STREET" + } + ], + "22": [ + { + "value": "Redact Signatures Redact", + "startOffset": 12, + "endOffset": 36, + "type": "ORG" + }, + { + "value": "Dilara Sonnenschein Signed", + "startOffset": 166, + "endOffset": 192, + "type": "ORG" + }, + { + "value": "Tobias M\u00fcller", + "startOffset": 197, + "endOffset": 210, + "type": "ORG" + } + ], + "23": [ + { + "value": "Redact Logo Redact", + "startOffset": 9, + "endOffset": 27, + "type": "ORG" + } + ] + } +} \ No newline at end of file