From 75d677942a5f866ac5332d7f2a3e877a19ceab11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Fri, 15 Sep 2023 13:11:11 +0200 Subject: [PATCH] DM-285: fix component drools execution --- .../entity/ManualChangeOverwrite.java | 4 ++ .../server/model/drools/RuleIdentifier.java | 19 ++----- .../v1/server/service/AnalyzeService.java | 56 +++++++++++-------- .../service/ComponentLogCreatorService.java | 8 +-- .../service/EntityChangeLogService.java | 4 +- .../service/RedactionChangeLogService.java | 4 +- .../service/RedactionLogCreatorService.java | 5 +- .../document/ComponentCreationService.java | 7 ++- .../document/ManualEntityCreationService.java | 2 +- .../v1/server/DocumineFloraTest.java | 5 +- .../drools/documine_flora_components.drl | 2 +- 11 files changed, 57 insertions(+), 59 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java index a6b0ee3e..76a21630 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java @@ -108,6 +108,10 @@ public class ManualChangeOverwrite { type = recategorization.getType(); } + if (manualChange instanceof ManualRedactionEntry redactionEntry) { + legalBasis = redactionEntry.getLegalBasis(); + } + descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass())); } changed = false; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java index 3dbf0fb3..2d09c566 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java @@ -1,9 +1,6 @@ package com.iqser.red.service.redaction.v1.server.model.drools; -import java.util.Arrays; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; import lombok.NonNull; @@ -35,18 +32,6 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { } - public static Set fromListOfIdentifiersString(String input) { - - return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet()); - } - - - private static Integer parseIntOrStar(String value) { - - return !value.equals("*") ? Integer.parseInt(value) : null; - } - - /** * This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier. * Therefore, to compare RuleIdentifiers one should always use the function matches. @@ -66,6 +51,10 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { @Override public String toString() { + if (type.name().isBlank() || type.name().isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); sb.append(type().name()); if (Objects.nonNull(unit())) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 025cf17a..7c2063c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -120,9 +120,15 @@ public class AnalyzeService { RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); - return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, entityLog, + return finalizeAnalysis(analyzeRequest, + startTime, + kieWrapperComponentRules, + entityLog, redactionLog, - document.getNumberOfPages(), dictionary.getVersion(), false, new HashSet<>(allFileAttributes)); + document.getNumberOfPages(), + dictionary.getVersion(), + false, + new HashSet<>(allFileAttributes)); } @@ -191,9 +197,7 @@ public class AnalyzeService { if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, - kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), - previousEntityLog, - previousRedactionLog, + kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog, previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, @@ -226,9 +230,7 @@ public class AnalyzeService { EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds); return finalizeAnalysis(analyzeRequest, - startTime, - .749.749 0 1 1 1.06-1.06z"/> addedFileAttributes, - EntityLogChanges entityLogChanges) { + EntityLogChanges entityLogChanges, + DictionaryVersion dictionaryVersion) { if (!kieWrapperComponentRules.isPresent()) { return; @@ -336,34 +343,37 @@ public class AnalyzeService { addedFileAttributes.stream().toList()); log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components); + ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), + components, + dictionaryVersion, + kieWrapperComponentRules.rulesVersion()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog); } - private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) { + private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, + EntityLog entityLog, + RedactionLog redactionLog, + DictionaryVersion dictionaryVersion) { + + EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); // TODO: remove redactionLog related stuff redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); - RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - redactionLog, - analyzeRequest.getAnalysisNumber()); + RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(previousRedactionLog, redactionLog, analyzeRequest.getAnalysisNumber()); entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages()); - EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - entityLog, - analyzeRequest.getAnalysisNumber()); + EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(previousEntityLog, entityLog, analyzeRequest.getAnalysisNumber()); log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java index 184c61d7..4a7d65dc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -12,12 +12,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators; @Service public class ComponentLogCreatorService { - public ComponentLog buildComponentLog(int analysisNumber, List components) { + public ComponentLog buildComponentLog(int analysisNumber, List components, DictionaryVersion dictionaryVersion, long rulesVersion) { List componentLogCategories = components.stream() .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) @@ -25,7 +26,7 @@ public class ComponentLogCreatorService { .stream() .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue())) .toList(); - return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1); + return new ComponentLog(analysisNumber, componentLogCategories, dictionaryVersion.getDossierVersion(), dictionaryVersion.getDossierTemplateVersion(), rulesVersion); } @@ -47,8 +48,7 @@ public class ComponentLogCreatorService { private ComponentEntityReference toComponentEntityReference(Entity entity) { - return ComponentEntityReference.builder() - .id(entity.getId()).value(entity.getValue()) + return ComponentEntityReference.builder().id(entity.getId()).value(entity.getValue()) .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)) .reason(entity.getReason()) .ruleIdentifier(entity.getMatchedRule()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index c4d74553..283a606a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -31,12 +31,10 @@ public class EntityChangeLogService { @Timed("redactmanager_computeChanges") - public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) { + public EntityLogChanges computeChanges(EntityLog previousEntityLog, EntityLog currentEntityLog, int analysisNumber) { long start = System.currentTimeMillis(); - EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId); - if (previousEntityLog == null) { currentEntityLog.getEntityLogEntry().forEach(entry -> { entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java index 74c4dfc4..d2c26f5f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java @@ -31,12 +31,10 @@ public class RedactionChangeLogService { @Timed("redactmanager_computeChanges") - public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) { + public RedactionLogChanges computeChanges(RedactionLog previousRedactionLog, RedactionLog currentRedactionLog, int analysisNumber) { long start = System.currentTimeMillis(); - RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); - if (previousRedactionLog == null) { currentRedactionLog.getRedactionLogEntry().forEach(entry -> { entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 622d87a5..589bfe0a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -286,9 +286,8 @@ public class RedactionLogCreatorService { private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { - return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), - (float) rectangle2D.getWidth(), - -(float) rectangle2D.getHeight(), + return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), + (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index dd948fc7..51c3c41d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -79,7 +79,9 @@ public class ComponentCreationService { public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection entities) { - entities.forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity))); + entities.stream() + .filter(entity -> !referencedEntities.contains(entity)) + .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity))); } @@ -91,8 +93,7 @@ public class ComponentCreationService { .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) .category(category) .value(value) - .transformation(transformation) - .references(new LinkedList<>(references))); + .transformation(transformation).references(new LinkedList<>(references)).build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java index 28aea8a8..1467096f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java @@ -118,7 +118,7 @@ public class ManualEntityCreationService { private Optional findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map> entitiesWithSameValue) { - List possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ROOT)); + List possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ENGLISH)); if (entityIdentifierValueNotFound(possibleEntities)) { log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index cd0b91c1..be26b3a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -30,7 +29,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; -import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -100,7 +98,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { @Test - @Disabled +// @Disabled public void testTopOfPage13InNotHeader() throws IOException { // Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf @@ -114,6 +112,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl index 54c02fd1..b6e6b2df 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -228,7 +228,7 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities." rule "X.0.0: merge duplicate component references" when $first: Component() - $duplicate: Component(this != $first, category == category, value == value) + $duplicate: Component(this != $first, category == $first.category, value == $first.value) then $first.getReferences().addAll($duplicate.getReferences()); retract($duplicate);