diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 5678254b..a4790e05 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.587.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index ddf0d5ab..fb76bd65 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,8 +16,8 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.592.0-RED10260.0" -val llmServiceVersion = "1.11.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" +val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 39c9abae..21e7c63b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -34,7 +34,7 @@ public class RedactionServiceSettings { private int dictionaryCacheExpireAfterAccessDays = 3; - private int droolsExecutionTimeoutSecs = 300; + private int droolsExecutionTimeoutSecs = 600; private boolean ruleExecutionSecured = true; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java index 5d1f8b17..fd08f339 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java @@ -82,4 +82,14 @@ public class NerEntities { LLM_NER } + + public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) { + + return switch (nerEntityEngine) { + case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER; + case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER; + }; + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index 3cc3cb3c..9929cc3c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -95,7 +95,9 @@ public interface IEntity { */ // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner. default boolean applied() { - + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isApplied(); + } return getManualOverwrite().getApplied() .orElse(getMatchedRule().isApplied()); } @@ -118,6 +120,9 @@ public interface IEntity { * @return True if ignored, false otherwise. */ default boolean ignored() { + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isIgnored(); + } return getManualOverwrite().getIgnored() .orElse(getMatchedRule().isIgnored()); @@ -130,8 +135,10 @@ public interface IEntity { * @return True if removed, false otherwise. */ default boolean removed() { - - return getManualOverwrite().getRemoved() + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isRemoved(); + } + return getManualOverwrite().getRemoved() .orElse(getMatchedRule().isRemoved()); } @@ -142,7 +149,9 @@ public interface IEntity { * @return True if resized, false otherwise. */ default boolean resized() { - + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isRemoved(); + } return getManualOverwrite().getResized() .orElse(false); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index a72a9adf..02f485c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -28,8 +28,9 @@ public final class MatchedRule implements Comparable { public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL"); public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP"); + public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN"); public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT"); - private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE); + private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, MANUAL_TYPE, DICTIONARY_TYPE); RuleIdentifier ruleIdentifier; @Builder.Default @@ -56,6 +57,10 @@ public final class MatchedRule implements Comparable { return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build(); } + public boolean isHigherPriorityThanManual() { + return (-1 < RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type())) && + (RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type()) < RULE_TYPE_PRIORITIES.indexOf(MANUAL_TYPE)); + } /** * Returns a modified instance of {@link MatchedRule} based on its applied status. diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 2d085604..e7aca7cb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -25,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService; import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -167,13 +168,23 @@ public class RedactionMessageReceiver { private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) { log.error("Failed to process analyze request: {}", analyzeRequest, e); - var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + + ErrorCode errorCode = null; + + if(e instanceof DroolsTimeoutException dre){ + if (!dre.isReported()){ + errorCode = ErrorCode.RULES_EXECUTION_TIMEOUT; + } else { + errorCode = ErrorCode.LOCKED_RULES; + } + } + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE, "redaction-service", - timestamp)); + errorCode)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java index f49f4056..91f80ede 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java @@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; @@ -38,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.knecon.fforesight.llm.service.LlmNerEntities; import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; @@ -285,8 +287,10 @@ public class AnalysisPreparationService { private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) { - return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities() + LlmNerEntities llmNerEntities = redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + return new NerEntities(llmNerEntities.getEntities() .stream() + //.filter(e -> allTypes.contains(e.getType())) .map(e -> new NerEntities.NerEntity(e.getValue(), new TextRange(e.getStartOffset(), e.getEndOffset()), e.getType(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 6f2eaf06..5e6485ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -20,6 +20,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index 8f16628a..1fdfda31 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -456,4 +456,12 @@ public class DictionaryService { return dossierDictRep != null; } + + public List getAllTypes(String dossierTemplateId, String dossierId) { + + List allTypes = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, null, false); + allTypes.addAll(dictionaryClient.getAllTypesForDossier(dossierId, null, false)); + return allTypes; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 421f4e3d..380a1cf7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -390,9 +390,6 @@ public class EntityLogCreatorService { Set engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>(); - if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) { - engines.add(Engine.MANUAL); - } return engines; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index a6c592b5..156ffb54 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -9,6 +9,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -1176,7 +1177,7 @@ public class EntityCreationService { */ public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException( + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException( "No entity present!")); } @@ -1192,7 +1193,11 @@ public class EntityCreationService { */ public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!")); + return byTextRangeWithEngine(nerEntity.textRange(), + type, + entityType, + semanticNode, + getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException("No entity present!")); } @@ -1206,7 +1211,7 @@ public class EntityCreationService { */ public Optional optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1221,7 +1226,7 @@ public class EntityCreationService { */ public Optional optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1240,7 +1245,7 @@ public class EntityCreationService { if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) { return Optional.empty(); } - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1264,7 +1269,7 @@ public class EntityCreationService { if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) { return Optional.empty(); } - return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1280,7 +1285,7 @@ public class EntityCreationService { public Stream combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) { return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1307,7 +1312,7 @@ public class EntityCreationService { int minPartsToCombine) { return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1336,7 +1341,7 @@ public class EntityCreationService { int minPartsToCombine) { return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1372,7 +1377,7 @@ public class EntityCreationService { maxDistanceBetweenParts, minPartsToCombine, minEssentialTypesCombined) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1411,7 +1416,7 @@ public class EntityCreationService { minPartsToCombine, minEssentialTypesCombined, confidence) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1440,7 +1445,7 @@ public class EntityCreationService { DocumentTree documentTree = node.getDocumentTree(); if (node.getEntities().contains(entity)) { - // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges + // If entity already exists, and it has a different text range, we add the text range to the list of duplicated text ranges node.getEntities() .stream()// .filter(e -> e.equals(entity))// @@ -1455,6 +1460,23 @@ public class EntityCreationService { } + private static Set getNerEngines(NerEntities nerEntities) { + + return getNerEngines(nerEntities.getNerEntityList() + .toArray(new NerEntities.NerEntity[0])); + } + + + private static Set getNerEngines(NerEntities.NerEntity... nerEntities) { + + return Arrays.stream(nerEntities) + .map(NerEntities.NerEntity::engine) + .map(NerEntities::mapToPrimaryEngine) + .collect(Collectors.toSet()); + } + + + private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) { entityToDuplicate.addTextRange(newTextRange); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index eb90ac88..355c6a73 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -123,7 +123,7 @@ public class ComponentDroolsExecutionService { logger.error(e, "Exception during rule execution"); kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { - throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.COMPONENT); } throw new RuntimeException(e); } catch (InterruptedException e) { @@ -131,7 +131,7 @@ public class ComponentDroolsExecutionService { kieSession.dispose(); throw new RuntimeException(e); } catch (TimeoutException e) { - throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.COMPONENT); } List resultingFileAttributes = getFileAttributes(kieSession); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index b5da9b1e..bbad6457 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -156,7 +156,7 @@ public class EntityDroolsExecutionService { logger.error(e, "Exception during rule execution"); kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { - throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY); } throw new RuntimeException(e); } catch (InterruptedException e) { @@ -164,7 +164,7 @@ public class EntityDroolsExecutionService { kieSession.dispose(); throw new RuntimeException(e); } catch (TimeoutException e) { - throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); + throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY); } List resultingFileAttributes = getFileAttributes(kieSession); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 1538c295..182fabfb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -6,6 +6,7 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi import java.io.File; import java.io.FileInputStream; import java.io.InputStream; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; @@ -31,7 +32,6 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.document.DocumentData; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; -import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.llm.service.LlmNerEntities; @@ -137,13 +137,13 @@ public class RedactionStorageService { } - @SneakyThrows public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) { componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog); } + @SneakyThrows public void updateEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { @@ -373,7 +373,12 @@ public class RedactionStorageService { public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class); + String objectId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES); + if (storageService.objectExists(TenantContext.getTenantId(), objectId)) { + return storageService.readJSONObject(TenantContext.getTenantId(), objectId, LlmNerEntities.class); + } else { + return LlmNerEntities.builder().entities(new ArrayList<>()).build(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java index e4ee9642..aa955c6c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java @@ -22,6 +22,14 @@ public class DroolsTimeoutException extends RuntimeException { } + public DroolsTimeoutException(String message, Throwable cause, boolean reported, RuleFileType ruleFileType) { + + super(message, cause); + this.reported = reported; + this.ruleFileType = ruleFileType; + } + + public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) { super(DROOLS_TIMEOUT_MESSAGE); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 430ab672..2f5d5a56 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -261,6 +261,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { } + @Disabled @Test public void testDoseMortalityExtraction() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 8dc9ad8a..b1eba29c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server; import static com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -45,11 +46,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; @@ -61,6 +64,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; @@ -2164,7 +2168,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { String assessment = "assessment"; dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment); reanlysisVersions.put(assessment, 1L); - when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L); analyzeService.reanalyze(request); @@ -2191,6 +2194,126 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { } + @Test + @SneakyThrows + @Order(3) + public void testLocalRemovalOfDictEntry() { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + + String pdfFile = "files/new/test_file.pdf"; + + String manualDictAddId = UUID.randomUUID().toString(); + String manualAddId = UUID.randomUUID().toString(); + String valueToAdd = "Crandu Seku Laku Meku"; + +// positions=[Rectangle(topLeft=Point(x=56.8, y=527.264), width=120.96, height=15.408, page=1)], + List fullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(527.264f).width(120.96f).height(15.408f).page(1).build()); + ManualRedactionEntry manualDictRedactionEntry = new ManualRedactionEntry(); + manualDictRedactionEntry.setAnnotationId(manualDictAddId); + manualDictRedactionEntry.setFileId("fileId"); + manualDictRedactionEntry.setUser("test"); + manualDictRedactionEntry.setType("CBI_author"); + manualDictRedactionEntry.setRectangle(false); + manualDictRedactionEntry.setAddToDictionary(true); + manualDictRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY); + manualDictRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualDictRedactionEntry.setValue(valueToAdd); + manualDictRedactionEntry.setReason("Dictionary Request"); + manualDictRedactionEntry.setPositions(fullPositions); + + var idUsedForRemoval = "4d92d86e7d70ab9bb5c0d554bfa3c7f0"; + var idRemoval = getIdRemoval(idUsedForRemoval); + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + Set entriesToAdd = Set.of(manualDictRedactionEntry); + Set entriesToRemove = Set.of(idRemoval); + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).idsToRemove(entriesToRemove).build()); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + request.setAnalysisNumber(0); + mockDictionaryCalls(1L); + + dossierDictionary.get(DICTIONARY_AUTHOR).add(valueToAdd); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L); + reanlysisVersions.put(valueToAdd, 0L); + + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertEquals(entityLog.getEntityLogEntry().size(), 4); + + EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval)) + .findFirst() + .get(); + assertEquals(entityLogEntry1.getState(), EntryState.IGNORED); + assertEquals(entityLogEntry1.getMatchedRule(), "CBI.0.3"); + assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL)); +//-----------force local + var processedDate = OffsetDateTime.now(); + manualDictRedactionEntry.setProcessedDate(processedDate); + idRemoval.setProcessedDate(processedDate); + + //[Rectangle(topLeft=Point(x=56.8, y=528.9), width=120.96001, height=12.642, page=1)] + List localfullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(528.9f).width(120.96f).height(12.642f).page(1).build()); + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(manualAddId); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setUser("test"); + manualRedactionEntry.setType("CBI_author"); + manualRedactionEntry.setRectangle(false); + manualRedactionEntry.setAddToDictionary(false); + manualRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setValue(valueToAdd); + manualRedactionEntry.setReason("Author found, removed by manual override"); + manualRedactionEntry.setSection("Header: This is my test"); + manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + manualRedactionEntry.setTextBefore("Lorem My Ipsum "); + manualRedactionEntry.setTextAfter("Crandu Seku Laku"); + manualRedactionEntry.setPositions(localfullPositions); + + ManualForceRedaction forceRequest = new ManualForceRedaction(); + forceRequest.setAnnotationId(manualAddId); + forceRequest.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + forceRequest.setUser("test"); + forceRequest.setRequestDate(OffsetDateTime.now()); + + Set forceRedactions = Set.of(forceRequest); + request.setManualRedactions(ManualRedactions.builder() + .entriesToAdd(Set.of(manualDictRedactionEntry, manualRedactionEntry)) + .idsToRemove(entriesToRemove) + .forceRedactions(forceRedactions) + .build()); + request.setAnalysisNumber(2); + + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + assertEquals(entityLog.getEntityLogEntry().size(), 5); + + entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval)) + .findFirst() + .get(); + var entityLogEntry2 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get(); + assertEquals(entityLogEntry1.getState(), EntryState.REMOVED); + assertEquals(entityLogEntry1.getMatchedRule(), "X.11.1"); + assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL)); + assertEquals(entityLogEntry2.getState(), EntryState.APPLIED); + assertEquals(entityLogEntry2.getMatchedRule(), "MAN.5.0"); + dossierDictionary.get(DICTIONARY_AUTHOR).remove(valueToAdd); + } + + @Test @SneakyThrows public void testDocumentDataFallback() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java index 75a77785..32fb8fbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java @@ -30,8 +30,8 @@ public class TextEntityTest { entity.apply("MAN.3.0", ""); entity.apply("MAN.3.3", ""); entity.skip("CBI.13.2", ""); - assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2"); - assertThat(entity.getMatchedRuleUnit()).isEqualTo(13); + assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("MAN.3.0"); + assertThat(entity.getMatchedRuleUnit()).isEqualTo(3); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf new file mode 100644 index 00000000..8daa5ae3 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf differ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta index 5705cc07..57e6e0dd 160000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta @@ -1 +1 @@ -Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421 +Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7