Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-6929' into 'master'

Browse Source 
RED-6929: fix acceptance tests/results

Closes RED-6929

See merge request redactmanager/redaction-service!42
This commit is contained in:
Kilian Schüttler 2023-07-10 15:14:08 +02:00
commit dfa66e99db
11 changed files with 35 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -30,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
@ -364,9 +365,7 @@ public class EntityCreationService {
}
addEntityToGraph(entity, node);
entity.addEngine(Engine.RULE);
if (kieSession != null) {
kieSession.insert(entity);
}
insertToKieSession(entity);
return Optional.of(entity);
}
@ -402,15 +401,24 @@ public class EntityCreationService {
mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved));
addEntityToGraph(mergedEntity, node);
kieSession.insert(mergedEntity);
insertToKieSession(mergedEntity);
return mergedEntity;
}
private void insertToKieSession(RedactionEntity mergedEntity) {
if (kieSession != null) {
kieSession.insert(mergedEntity);
}
}
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
var entity = forceByBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
entity.addEngine(Engine.NER);
insertToKieSession(entity);
return entity;
}
@ -419,10 +427,20 @@ public class EntityCreationService {
var entity = forceByBoundary(nerEntity.boundary(), type, entityType, semanticNode);
entity.addEngine(Engine.NER);
insertToKieSession(entity);
return entity;
}
public Stream<RedactionEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> forceByBoundary(boundary, type, entityType, semanticNode))
.peek(entity -> entity.addEngine(Engine.NER))
.peek(this::insertToKieSession);
}
public RedactionEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCell.getBoundary().start(), tableCell.getBoundary().start()), type, entityType);

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java
View File

@ -7,8 +7,6 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
@ -18,10 +16,11 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.te
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@UtilityClass
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class NerEntitiesAdapter {

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -64,7 +64,6 @@ public class AnalyzeService {
private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
DictionaryService dictionaryService;
NerEntitiesAdapter nerEntitiesAdapter;
DroolsExecutionService droolsExecutionService;
EntityRedactionService entityRedactionService;
RedactionLogCreatorService redactionLogCreatorService;
@ -310,7 +309,7 @@ public class AnalyzeService {
if (redactionServiceSettings.isNerServiceEnabled()) {
NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel);
nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
} else {
nerEntities = new NerEntities(Collections.emptyList());
}
@ -330,7 +329,7 @@ public class AnalyzeService {
NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {
nerEntities = nerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), document);
nerEntities = NerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), document);
} else {
nerEntities = new NerEntities(Collections.emptyList());
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -29,7 +29,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import io.micrometer.core.annotation.Timed;
@ -48,7 +47,6 @@ public class DroolsExecutionService {
Map<String, KieContainer> kieContainers = new HashMap<>();
Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
EntityEnrichmentService entityEnrichmentService;
NerEntitiesAdapter nerEntitiesAdapter;
public KieContainer getKieContainer(String dossierTemplateId) {
@ -91,7 +89,6 @@ public class DroolsExecutionService {
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
kieSession.setGlobal("dictionary", dictionary);
kieSession.setGlobal("nerEntitiesAdapter", nerEntitiesAdapter);
kieSession.insert(document);
document.getEntities().forEach(kieSession::insert);

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java
View File

@ -36,8 +36,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
@Autowired
private NerEntitiesAdapter nerEntitiesAdapter;
@Autowired
private EntityEnrichmentService entityEnrichmentService;
private EntityCreationService entityCreationService;
@ -71,7 +69,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
ClassPathResource resource = new ClassPathResource(filePath);
PDDocument pdDocument = PDDocument.load(resource.getInputStream());
Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
Stream<NerEntities.NerEntity> unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
.getNerEntityList()
.stream()
.filter(e -> !e.type().equals("CBI_author"));
@ -105,9 +103,9 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
log.info("Graph built");
NerEntitiesModel nerEntitiesModel = parseNerEntities(nerEntitiesFilePath);
log.info("Parsed NerEntitiesModel");
NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
log.info("Validated and mapped");
List<Boundary> nerEntityBoundaries = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList();
List<Boundary> nerEntityBoundaries = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList();
log.info("Combined to CBI_address");
List<RedactionEntity> cbiAddressEntities = nerEntityBoundaries.stream()
.map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, document))
@ -119,7 +117,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
ClassPathResource resource = new ClassPathResource(filePath);
PDDocument pdDocument = PDDocument.load(resource.getInputStream());
List<RedactionEntity> validatedEntities = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
List<RedactionEntity> validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document)
.getNerEntityList()
.stream()
.map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, document))
@ -143,10 +141,10 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
private List<NerEntities.NerEntity> validateAndCombine(NerEntitiesModel nerEntitiesModel, Document document) {
NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList();
Stream<NerEntities.NerEntity> cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
Stream<NerEntities.NerEntity> cbiAddress = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> new NerEntities.NerEntity(document.getTextBlock().subSequence(boundary).toString(), boundary, "CBI_address"));
return Stream.concat(cbiAuthors.stream(), cbiAddress).toList();

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -39,14 +39,12 @@ import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionE
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
@ -477,9 +475,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> entity.addEngine(Engine.NER));
entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
end

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -39,14 +39,12 @@ import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionE
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
@ -1136,9 +1134,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> entity.addEngine(Engine.NER));
entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -45,7 +45,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
//------------------------------------ queries ------------------------------------

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -39,7 +39,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
//------------------------------------ queries ------------------------------------

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -39,14 +39,12 @@ import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionE
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
@ -861,9 +859,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> entity.addEngine(Engine.NER));
entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -40,7 +40,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------