Merge branch 'master' into feature/RED-10290

# Conflicts:
#	redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.587.0"
+val persistenceServiceVersion = "2.612.0-RED10072.1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,8 +16,8 @@ val layoutParserVersion = "0.181.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.592.0-RED10260.0"
+val persistenceServiceVersion = "2.612.0-RED10072.1"
-val llmServiceVersion = "1.11.0"
+val llmServiceVersion = "1.20.0-RED10072.2"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java

@@ -34,7 +34,7 @@ public class RedactionServiceSettings {
 
     private int dictionaryCacheExpireAfterAccessDays = 3;
 
-    private int droolsExecutionTimeoutSecs = 300;
+    private int droolsExecutionTimeoutSecs = 600;
 
     private boolean ruleExecutionSecured = true;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java

@@ -82,4 +82,14 @@ public class NerEntities {
         LLM_NER
     }
 
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) {
+
+        return switch (nerEntityEngine) {
+            case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER;
+        };
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java

@@ -95,7 +95,9 @@ public interface IEntity {
      */
     // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
     default boolean applied() {
-
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isApplied();
+        }
         return getManualOverwrite().getApplied()
                 .orElse(getMatchedRule().isApplied());
     }
@@ -118,6 +120,9 @@ public interface IEntity {
      * @return True if ignored, false otherwise.
      */
     default boolean ignored() {
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isIgnored();
+        }
 
         return getManualOverwrite().getIgnored()
                 .orElse(getMatchedRule().isIgnored());
@@ -130,8 +135,10 @@ public interface IEntity {
      * @return True if removed, false otherwise.
      */
     default boolean removed() {
-
+       if (this.getMatchedRule().isHigherPriorityThanManual()) {
-        return getManualOverwrite().getRemoved()
+           return getMatchedRule().isRemoved();
+       }
+       return getManualOverwrite().getRemoved()
                 .orElse(getMatchedRule().isRemoved());
     }
 
@@ -142,7 +149,9 @@ public interface IEntity {
      * @return True if resized, false otherwise.
      */
     default boolean resized() {
-
+        if (this.getMatchedRule().isHigherPriorityThanManual()) {
+            return getMatchedRule().isRemoved();
+        }
         return getManualOverwrite().getResized()
                 .orElse(false);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -28,8 +28,9 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
     public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
     public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
+    public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN");
     public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, MANUAL_TYPE, DICTIONARY_TYPE);
 
     RuleIdentifier ruleIdentifier;
     @Builder.Default
@@ -56,6 +57,10 @@ public final class MatchedRule implements Comparable<MatchedRule> {
         return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build();
     }
 
+    public boolean isHigherPriorityThanManual() {
+        return (-1 < RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type())) &&
+                       (RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type()) < RULE_TYPE_PRIORITIES.indexOf(MANUAL_TYPE));
+    }
 
     /**
      * Returns a modified instance of {@link MatchedRule} based on its applied status.

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -25,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService;
 import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -167,13 +168,23 @@ public class RedactionMessageReceiver {
     private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
 
         log.error("Failed to process analyze request: {}", analyzeRequest, e);
-        var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+
+        ErrorCode errorCode = null;
+
+        if(e instanceof DroolsTimeoutException dre){
+            if (!dre.isReported()){
+                errorCode = ErrorCode.RULES_EXECUTION_TIMEOUT;
+            } else {
+                errorCode = ErrorCode.LOCKED_RULES;
+            }
+        }
+
         fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                                                         analyzeRequest.getFileId(),
                                                         new FileErrorInfo(e.getMessage(),
                                                                           priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE,
                                                                           "redaction-service",
-                                                                          timestamp));
+                                                                          errorCode));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java

@@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
@@ -38,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS
 import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.knecon.fforesight.llm.service.LlmNerEntities;
 
 import io.micrometer.core.annotation.Timed;
 import io.micrometer.observation.annotation.Observed;
@@ -285,8 +287,10 @@ public class AnalysisPreparationService {
 
     private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) {
 
-        return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities()
+        LlmNerEntities llmNerEntities = redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        return new NerEntities(llmNerEntities.getEntities()
                                        .stream()
+                                       //.filter(e -> allTypes.contains(e.getType()))
                                        .map(e -> new NerEntities.NerEntity(e.getValue(),
                                                                            new TextRange(e.getStartOffset(), e.getEndOffset()),
                                                                            e.getType(),

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -20,6 +20,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -456,4 +456,12 @@ public class DictionaryService {
         return dossierDictRep != null;
     }
 
+
+    public List<Type> getAllTypes(String dossierTemplateId, String dossierId) {
+
+        List<Type> allTypes = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, null, false);
+        allTypes.addAll(dictionaryClient.getAllTypesForDossier(dossierId, null, false));
+        return allTypes;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -390,9 +390,6 @@ public class EntityLogCreatorService {
 
         Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
 
-        if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
-            engines.add(Engine.MANUAL);
-        }
         return engines;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -9,6 +9,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC
 import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -1176,7 +1177,7 @@ public class EntityCreationService {
      */
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException(
                 "No entity present!"));
     }
 
@@ -1192,7 +1193,11 @@ public class EntityCreationService {
      */
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
+        return byTextRangeWithEngine(nerEntity.textRange(),
+                                     type,
+                                     entityType,
+                                     semanticNode,
+                                     getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException("No entity present!"));
     }
 
 
@@ -1206,7 +1211,7 @@ public class EntityCreationService {
      */
     public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1221,7 +1226,7 @@ public class EntityCreationService {
      */
     public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1240,7 +1245,7 @@ public class EntityCreationService {
         if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
             return Optional.empty();
         }
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1264,7 +1269,7 @@ public class EntityCreationService {
         if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
             return Optional.empty();
         }
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1280,7 +1285,7 @@ public class EntityCreationService {
     public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
 
         return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1307,7 +1312,7 @@ public class EntityCreationService {
                                                  int minPartsToCombine) {
 
         return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1336,7 +1341,7 @@ public class EntityCreationService {
                                                  int minPartsToCombine) {
 
         return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1372,7 +1377,7 @@ public class EntityCreationService {
                                                                     maxDistanceBetweenParts,
                                                                     minPartsToCombine,
                                                                     minEssentialTypesCombined)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1411,7 +1416,7 @@ public class EntityCreationService {
                                                                                   minPartsToCombine,
                                                                                   minEssentialTypesCombined,
                                                                                   confidence)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1440,7 +1445,7 @@ public class EntityCreationService {
 
         DocumentTree documentTree = node.getDocumentTree();
         if (node.getEntities().contains(entity)) {
-            // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
+            // If entity already exists, and it has a different text range, we add the text range to the list of duplicated text ranges
             node.getEntities()
                     .stream()//
                     .filter(e -> e.equals(entity))//
@@ -1455,6 +1460,23 @@ public class EntityCreationService {
     }
 
 
+    private static Set<Engine> getNerEngines(NerEntities nerEntities) {
+
+        return getNerEngines(nerEntities.getNerEntityList()
+                                     .toArray(new NerEntities.NerEntity[0]));
+    }
+
+
+    private static Set<Engine> getNerEngines(NerEntities.NerEntity... nerEntities) {
+
+        return Arrays.stream(nerEntities)
+                .map(NerEntities.NerEntity::engine)
+                .map(NerEntities::mapToPrimaryEngine)
+                .collect(Collectors.toSet());
+    }
+
+
+
     private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
 
         entityToDuplicate.addTextRange(newTextRange);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -123,7 +123,7 @@ public class ComponentDroolsExecutionService {
             logger.error(e, "Exception during rule execution");
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {
-                throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
+                throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.COMPONENT);
             }
             throw new RuntimeException(e);
         } catch (InterruptedException e) {
@@ -131,7 +131,7 @@ public class ComponentDroolsExecutionService {
             kieSession.dispose();
             throw new RuntimeException(e);
         } catch (TimeoutException e) {
-            throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
+            throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.COMPONENT);
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -156,7 +156,7 @@ public class EntityDroolsExecutionService {
             logger.error(e, "Exception during rule execution");
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {
-                throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
+                throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY);
             }
             throw new RuntimeException(e);
         } catch (InterruptedException e) {
@@ -164,7 +164,7 @@ public class EntityDroolsExecutionService {
             kieSession.dispose();
             throw new RuntimeException(e);
         } catch (TimeoutException e) {
-            throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
+            throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY);
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -6,6 +6,7 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
@@ -31,7 +32,6 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
-import com.iqser.red.storage.commons.exception.StorageException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.llm.service.LlmNerEntities;
@@ -137,13 +137,13 @@ public class RedactionStorageService {
     }
 
 
-
     @SneakyThrows
     public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) {
 
         componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog);
     }
 
+
     @SneakyThrows
     public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
 
@@ -373,7 +373,12 @@ public class RedactionStorageService {
 
     public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) {
 
-        return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class);
+        String objectId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES);
+        if (storageService.objectExists(TenantContext.getTenantId(), objectId)) {
+            return storageService.readJSONObject(TenantContext.getTenantId(), objectId, LlmNerEntities.class);
+        } else {
+            return LlmNerEntities.builder().entities(new ArrayList<>()).build();
+        }
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java

@@ -22,6 +22,14 @@ public class DroolsTimeoutException extends RuntimeException {
     }
 
 
+    public DroolsTimeoutException(String message, Throwable cause, boolean reported, RuleFileType ruleFileType) {
+
+        super(message, cause);
+        this.reported = reported;
+        this.ruleFileType = ruleFileType;
+    }
+
+
     public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) {
 
         super(DROOLS_TIMEOUT_MESSAGE);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -261,6 +261,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Disabled
     @Test
     public void testDoseMortalityExtraction() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
 import static com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -45,11 +46,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -61,6 +64,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
@@ -2164,7 +2168,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
         String assessment = "assessment";
         dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment);
         reanlysisVersions.put(assessment, 1L);
-        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L);
 
         analyzeService.reanalyze(request);
 
@@ -2191,6 +2194,126 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    @Order(3)
+    public void testLocalRemovalOfDictEntry() {
+
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+
+        String pdfFile = "files/new/test_file.pdf";
+
+        String manualDictAddId = UUID.randomUUID().toString();
+        String manualAddId = UUID.randomUUID().toString();
+        String valueToAdd = "Crandu Seku Laku Meku";
+
+//        positions=[Rectangle(topLeft=Point(x=56.8, y=527.264), width=120.96, height=15.408, page=1)],
+        List<Rectangle> fullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(527.264f).width(120.96f).height(15.408f).page(1).build());
+        ManualRedactionEntry manualDictRedactionEntry = new ManualRedactionEntry();
+        manualDictRedactionEntry.setAnnotationId(manualDictAddId);
+        manualDictRedactionEntry.setFileId("fileId");
+        manualDictRedactionEntry.setUser("test");
+        manualDictRedactionEntry.setType("CBI_author");
+        manualDictRedactionEntry.setRectangle(false);
+        manualDictRedactionEntry.setAddToDictionary(true);
+        manualDictRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY);
+        manualDictRedactionEntry.setRequestDate(OffsetDateTime.now());
+        manualDictRedactionEntry.setValue(valueToAdd);
+        manualDictRedactionEntry.setReason("Dictionary Request");
+        manualDictRedactionEntry.setPositions(fullPositions);
+
+        var idUsedForRemoval = "4d92d86e7d70ab9bb5c0d554bfa3c7f0";
+        var idRemoval = getIdRemoval(idUsedForRemoval);
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        Set<ManualRedactionEntry> entriesToAdd = Set.of(manualDictRedactionEntry);
+        Set<IdRemoval> entriesToRemove = Set.of(idRemoval);
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).idsToRemove(entriesToRemove).build());
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        request.setAnalysisNumber(0);
+        mockDictionaryCalls(1L);
+
+        dossierDictionary.get(DICTIONARY_AUTHOR).add(valueToAdd);
+        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L);
+        reanlysisVersions.put(valueToAdd, 0L);
+
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertEquals(entityLog.getEntityLogEntry().size(), 4);
+
+        EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval))
+                .findFirst()
+                .get();
+        assertEquals(entityLogEntry1.getState(), EntryState.IGNORED);
+        assertEquals(entityLogEntry1.getMatchedRule(), "CBI.0.3");
+        assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL));
+//-----------force local
+        var processedDate = OffsetDateTime.now();
+        manualDictRedactionEntry.setProcessedDate(processedDate);
+        idRemoval.setProcessedDate(processedDate);
+
+        //[Rectangle(topLeft=Point(x=56.8, y=528.9), width=120.96001, height=12.642, page=1)]
+        List<Rectangle> localfullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(528.9f).width(120.96f).height(12.642f).page(1).build());
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(manualAddId);
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setUser("test");
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setRectangle(false);
+        manualRedactionEntry.setAddToDictionary(false);
+        manualRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY);
+        manualRedactionEntry.setRequestDate(OffsetDateTime.now());
+        manualRedactionEntry.setValue(valueToAdd);
+        manualRedactionEntry.setReason("Author found, removed by manual override");
+        manualRedactionEntry.setSection("Header: This is my test");
+        manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+        manualRedactionEntry.setTextBefore("Lorem My Ipsum ");
+        manualRedactionEntry.setTextAfter("Crandu Seku Laku");
+        manualRedactionEntry.setPositions(localfullPositions);
+
+        ManualForceRedaction forceRequest = new ManualForceRedaction();
+        forceRequest.setAnnotationId(manualAddId);
+        forceRequest.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+        forceRequest.setUser("test");
+        forceRequest.setRequestDate(OffsetDateTime.now());
+
+        Set<ManualForceRedaction> forceRedactions = Set.of(forceRequest);
+        request.setManualRedactions(ManualRedactions.builder()
+                                            .entriesToAdd(Set.of(manualDictRedactionEntry, manualRedactionEntry))
+                                            .idsToRemove(entriesToRemove)
+                                            .forceRedactions(forceRedactions)
+                                            .build());
+        request.setAnalysisNumber(2);
+
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertEquals(entityLog.getEntityLogEntry().size(), 5);
+
+        entityLogEntry1 = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval))
+                .findFirst()
+                .get();
+        var entityLogEntry2 = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                .findFirst()
+                .get();
+        assertEquals(entityLogEntry1.getState(), EntryState.REMOVED);
+        assertEquals(entityLogEntry1.getMatchedRule(), "X.11.1");
+        assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL));
+        assertEquals(entityLogEntry2.getState(), EntryState.APPLIED);
+        assertEquals(entityLogEntry2.getMatchedRule(), "MAN.5.0");
+        dossierDictionary.get(DICTIONARY_AUTHOR).remove(valueToAdd);
+    }
+
+
     @Test
     @SneakyThrows
     public void testDocumentDataFallback() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java

@@ -30,8 +30,8 @@ public class TextEntityTest {
         entity.apply("MAN.3.0", "");
         entity.apply("MAN.3.3", "");
         entity.skip("CBI.13.2", "");
-        assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2");
+        assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("MAN.3.0");
-        assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
+        assertThat(entity.getMatchedRuleUnit()).isEqualTo(3);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta

@@ -1 +1 @@
-Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421
+Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7