Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

DM-285: fix string NPE

Browse Source
This commit is contained in:
Kilian Schüttler 2023-09-15 15:09:01 +02:00 committed by Dominique Eifländer
parent 636066e8e7
commit 39b754e5b1
10 changed files with 61 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java
View File

@ -35,7 +35,7 @@ public interface IEntity {
default String value() {
return getManualOverwrite().getValue().orElse(getValue());
return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue());
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -43,7 +43,7 @@ public class DictionarySearchService {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node))
.map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node))
.peek(entity -> entity.setDictionaryEntry(true))
.peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
.forEach(entity -> entity.addEngine(Engine.DICTIONARY));

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -118,8 +118,7 @@ public class EntityLogCreatorService {
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
.legalBasis(manualEntity.legalBasis()).value(manualEntity.value())
.type(type)
.state(buildEntryState(manualEntity))
.entryType(buildEntryType(manualEntity, isHint))
@ -145,7 +144,7 @@ public class EntityLogCreatorService {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return EntityLogEntry.builder()
.id(image.getId())
.id(image.getId()).value(image.value())
.color(getColor(imageType, dossierTemplateId, image.applied()))
.type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions())

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -529,7 +529,7 @@ public class EntityCreationService {
}
public TextEntity forceByBoundary(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
@ -598,7 +598,7 @@ public class EntityCreationService {
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
var entity = forceByBoundary(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode);
var entity = forceByTextRange(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode);
entity.addEngine(Engine.NER);
insertToKieSession(entity);
return entity;
@ -607,7 +607,7 @@ public class EntityCreationService {
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
var entity = forceByBoundary(nerEntity.textRange(), type, entityType, semanticNode);
var entity = forceByTextRange(nerEntity.textRange(), type, entityType, semanticNode);
entity.addEngine(Engine.NER);
insertToKieSession(entity);
return entity;
@ -616,8 +616,7 @@ public class EntityCreationService {
public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> forceByBoundary(boundary, type, entityType, semanticNode))
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).map(boundary -> forceByTextRange(boundary, type, entityType, semanticNode))
.peek(entity -> entity.addEngine(Engine.NER))
.peek(this::insertToKieSession);
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java
View File

@ -95,7 +95,7 @@ public class ManualEntityCreationService {
*/
private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
TextEntity correctEntity = entityCreationService.forceByBoundary(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
if (manualEntity.isApplied()) {
correctEntity.apply(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis());
@ -166,8 +166,7 @@ public class ManualEntityCreationService {
SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream()
.map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node))
.stream().map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
.collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -104,7 +104,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
// Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf
// TODO unify logic
AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 15_Toxicidade Oral Aguda.pdf");
AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -68,7 +67,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -215,7 +213,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Test
public void titleExtraction() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
@ -591,6 +589,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
}
public void testRedactionLogAndEntityLogEquality() {
}
@Test
public void testRemovePublishedInformations() throws IOException {

31
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.manualchanges;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertTrue;
@ -16,6 +15,9 @@ import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -41,8 +43,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@ -55,14 +57,13 @@ import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -114,7 +115,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
loadDictionaryForTest();
loadTypeForTest();
loadNerForTest();
@ -296,6 +296,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
var mouses = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals("vertebrate")).filter(entry -> entry.getValue().equals("Mouse")).toList();
var recategorizations = mouses.stream()
.map(mouse -> ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.type("published_information")
.annotationId(mouse.getId())
.fileId(TEST_FILE_ID)
.build())
.toList();
assertEquals("CBI.3.2", asyaLyon.getMatchedRule());
assertEquals("No vertebrate found", asyaLyon.getReason());
@ -308,7 +320,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.build();
request.setManualRedactions(new ManualRedactions());
request.getManualRedactions().setRecategorizations(Set.of(recategorization));
request.getManualRedactions()
.setRecategorizations(Stream.of(Stream.of(recategorization), recategorizations.stream()).flatMap(Function.identity()).collect(Collectors.toSet()));
analyzeService.reanalyze(request);
RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
@ -324,6 +337,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
for (RedactionLogEntry mouse : mouses) {
assertEquals("published_information", redactionLog2.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(mouse.getId())).findFirst().get().getType());
}
assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
}

40
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -91,7 +91,7 @@ rule "H.0.1: Ignore Table of Contents"
.forEach(node -> retract(node))
);
end
/*
@ -260,9 +260,9 @@ rule "DOC.1.3: Guidelines"
rule "DOC.3.2: Experimental Completion Date"
salience 10
when
when
$section: Section(onPage(1) && (containsString("STUDY COMPLETED ON") || containsString("STUDY COMPLETION DATE") || containsString("Report completion date") || containsString("Date of Report") || containsString("AMENDMENT COMPLETION DATE") || containsString("AMENDMENT COMPLETED ON")))
then
entityCreationService.byRegex("STUDY COMPLETED ON (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> {
entity.apply("DOC.3.2", "Experimental end date found", "n-a");
@ -341,7 +341,7 @@ rule "DOC.5.0: Strain"
rule "DOC.35.0: Sex"
when
$section: Section(
(
anyHeadlineContainsStringIgnoreCase("animal")
@ -436,7 +436,7 @@ rule "DOC.7.2: study title by document structure"
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("DATA REQUIREMENT", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AUTHOR(S)", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AUTHOR", $page.getMainBodyTextBlock()));
entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "title", EntityType.ENTITY, document).forEach(entity -> {
entity.apply("DOC.7.2", "Study title found", "n-a");
});
@ -452,7 +452,7 @@ rule "DOC.8.1: Performing Laboratory"
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("PERFORMING LABORATORY", $page.getMainBodyTextBlock()));
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITIES", $page.getMainBodyTextBlock()));
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITY", $page.getMainBodyTextBlock()));
List<TextRange> stopBoundaries = new LinkedList<>();
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("LABORATORY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock()));
@ -472,13 +472,13 @@ rule "DOC.8.2: Summary Methods"
$headline: Headline(containsString("1.1. METHODS"))
then
entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "summary_methods", EntityType.ENTITY)
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("Page"))
.filter(e -> !e.getValue().startsWith("2. INTRODUCTION"))
.filter(e -> !e.getValue().startsWith("BASF"))
.filter(e -> !e.getValue().startsWith("The Chemical Company"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.forEach(entity -> entity.apply("DOC.8.2", "Summary Methods found", "n-a"));
end
@ -488,12 +488,12 @@ rule "DOC.8.3: Summary Observations Laboratory"
then
entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "summary_observations", EntityType.ENTITY)
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("Page"))
.filter(e -> !e.getValue().startsWith("2. INTRODUCTION"))
.filter(e -> !e.getValue().startsWith("BASF"))
.filter(e -> !e.getValue().startsWith("The Chemical Company"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.forEach(entity -> entity.apply("DOC.8.3", "Summary Observations found", "n-a"));
end
@ -505,12 +505,12 @@ rule "DOC.8.5: Summary Results"
then
entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "summary_results", EntityType.ENTITY)
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("Page"))
.filter(e -> !e.getValue().startsWith("2. INTRODUCTION"))
.filter(e -> !e.getValue().startsWith("BASF"))
.filter(e -> !e.getValue().startsWith("The Chemical Company"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.forEach(entity -> {
entity.apply("DOC.8.5", "Summary Results", "n-a");
});
@ -522,12 +522,12 @@ rule "DOC.8.6: Summary Results 2"
then
entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "summary_results", EntityType.ENTITY)
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("Page"))
.filter(e -> !e.getValue().startsWith("2. INTRODUCTION"))
.filter(e -> !e.getValue().startsWith("BASF"))
.filter(e -> !e.getValue().startsWith("The Chemical Company"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.filter(e -> !e.getValue().startsWith("The Chemical Company"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.forEach(entity -> entity.apply("DOC.8.6", "Summary Results", "n-a"));
end
@ -538,13 +538,13 @@ rule "DOC.8.4: Summary Conclusion"
$headline: Headline(containsString("1.4. CONCLUSION") || containsString("1.3. CONCLUSION"))
then
entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "study_conclusion", EntityType.ENTITY)
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().contains("Report; Project No"))
.filter(e -> !e.getValue().startsWith("This document"))
.filter(e -> !e.getValue().startsWith("Page"))
.filter(e -> !e.getValue().startsWith("2. INTRODUCTION"))
.filter(e -> !e.getValue().startsWith("BASF"))
.filter(e -> !e.getValue().startsWith("The Chemical Company"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.filter(e -> !e.getValue().startsWith("We create chemistry"))
.forEach(entity -> entity.apply("DOC.8.4", "Summary Conculsion found", "n-a"));
end
@ -562,7 +562,7 @@ rule "DOC.9.0: GLP Study"
|| containsString("STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION")
|| containsString("GLP CERTIFICATE (FROM THE COMPETENT AUTHORITY)")
|| containsString("GLP COMPLIANCE STATEMENT")
|| containsString("GLP STATEMENT")
|| containsString("GLP STATEMENT")
)
then
entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> {

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Documine/Flora/402Study.pdf Normal file
View File

Binary file not shown.