Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'master' into feature/RED-10290

Browse Source 
# Conflicts:
#	redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
This commit is contained in:
maverickstuder 2024-11-07 15:00:29 +01:00
commit ab1ab0a50d
21 changed files with 242 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -4,7 +4,7 @@ plugins {
}
description = "redaction-service-api-v1"
val persistenceServiceVersion = "2.587.0"
val persistenceServiceVersion = "2.612.0-RED10072.1"
dependencies {
implementation("org.springframework:spring-web:6.0.12")

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,8 +16,8 @@ val layoutParserVersion = "0.181.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.592.0-RED10260.0"
val llmServiceVersion = "1.11.0"
val persistenceServiceVersion = "2.612.0-RED10072.1"
val llmServiceVersion = "1.20.0-RED10072.2"
val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4"
val testContainersVersion = "1.19.7"

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java
View File

@ -34,7 +34,7 @@ public class RedactionServiceSettings {
private int dictionaryCacheExpireAfterAccessDays = 3;
private int droolsExecutionTimeoutSecs = 300;
private int droolsExecutionTimeoutSecs = 600;
private boolean ruleExecutionSecured = true;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java
View File

@ -82,4 +82,14 @@ public class NerEntities {
LLM_NER
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) {
return switch (nerEntityEngine) {
case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER;
};
}
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java
View File

@ -95,7 +95,9 @@ public interface IEntity {
*/
// Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
default boolean applied() {
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isApplied();
}
return getManualOverwrite().getApplied()
.orElse(getMatchedRule().isApplied());
}
@ -118,6 +120,9 @@ public interface IEntity {
* @return True if ignored, false otherwise.
*/
default boolean ignored() {
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isIgnored();
}
return getManualOverwrite().getIgnored()
.orElse(getMatchedRule().isIgnored());
@ -130,8 +135,10 @@ public interface IEntity {
* @return True if removed, false otherwise.
*/
default boolean removed() {
return getManualOverwrite().getRemoved()
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isRemoved();
}
return getManualOverwrite().getRemoved()
.orElse(getMatchedRule().isRemoved());
}
@ -142,7 +149,9 @@ public interface IEntity {
* @return True if resized, false otherwise.
*/
default boolean resized() {
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isRemoved();
}
return getManualOverwrite().getResized()
.orElse(false);
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -28,8 +28,9 @@ public final class MatchedRule implements Comparable<MatchedRule> {
public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN");
public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, MANUAL_TYPE, DICTIONARY_TYPE);
RuleIdentifier ruleIdentifier;
@Builder.Default
@ -56,6 +57,10 @@ public final class MatchedRule implements Comparable<MatchedRule> {
return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build();
}
public boolean isHigherPriorityThanManual() {
return (-1 < RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type())) &&
(RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type()) < RULE_TYPE_PRIORITIES.indexOf(MANUAL_TYPE));
}
/**
* Returns a modified instance of {@link MatchedRule} based on its applied status.

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java
View File

@ -25,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -167,13 +168,23 @@ public class RedactionMessageReceiver {
private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
log.error("Failed to process analyze request: {}", analyzeRequest, e);
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
ErrorCode errorCode = null;
if(e instanceof DroolsTimeoutException dre){
if (!dre.isReported()){
errorCode = ErrorCode.RULES_EXECUTION_TIMEOUT;
} else {
errorCode = ErrorCode.LOCKED_RULES;
}
}
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
new FileErrorInfo(e.getMessage(),
priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE,
"redaction-service",
timestamp));
errorCode));
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
@ -38,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.knecon.fforesight.llm.service.LlmNerEntities;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
@ -285,8 +287,10 @@ public class AnalysisPreparationService {
private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) {
return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities()
LlmNerEntities llmNerEntities = redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
return new NerEntities(llmNerEntities.getEntities()
.stream()
//.filter(e -> allTypes.contains(e.getType()))
.map(e -> new NerEntities.NerEntity(e.getValue(),
new TextRange(e.getStartOffset(), e.getEndOffset()),
e.getType(),

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -20,6 +20,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
View File

@ -456,4 +456,12 @@ public class DictionaryService {
return dossierDictRep != null;
}
public List<Type> getAllTypes(String dossierTemplateId, String dossierId) {
List<Type> allTypes = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, null, false);
allTypes.addAll(dictionaryClient.getAllTypesForDossier(dossierId, null, false));
return allTypes;
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -390,9 +390,6 @@ public class EntityLogCreatorService {
Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
engines.add(Engine.MANUAL);
}
return engines;
}

46
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -9,6 +9,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@ -1176,7 +1177,7 @@ public class EntityCreationService {
*/
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException(
"No entity present!"));
}
@ -1192,7 +1193,11 @@ public class EntityCreationService {
*/
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
return byTextRangeWithEngine(nerEntity.textRange(),
type,
entityType,
semanticNode,
getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException("No entity present!"));
}
@ -1206,7 +1211,7 @@ public class EntityCreationService {
*/
public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1221,7 +1226,7 @@ public class EntityCreationService {
*/
public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1240,7 +1245,7 @@ public class EntityCreationService {
if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
return Optional.empty();
}
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1264,7 +1269,7 @@ public class EntityCreationService {
if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
return Optional.empty();
}
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1280,7 +1285,7 @@ public class EntityCreationService {
public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1307,7 +1312,7 @@ public class EntityCreationService {
int minPartsToCombine) {
return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1336,7 +1341,7 @@ public class EntityCreationService {
int minPartsToCombine) {
return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1372,7 +1377,7 @@ public class EntityCreationService {
maxDistanceBetweenParts,
minPartsToCombine,
minEssentialTypesCombined)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1411,7 +1416,7 @@ public class EntityCreationService {
minPartsToCombine,
minEssentialTypesCombined,
confidence)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1440,7 +1445,7 @@ public class EntityCreationService {
DocumentTree documentTree = node.getDocumentTree();
if (node.getEntities().contains(entity)) {
// If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
// If entity already exists, and it has a different text range, we add the text range to the list of duplicated text ranges
node.getEntities()
.stream()//
.filter(e -> e.equals(entity))//
@ -1455,6 +1460,23 @@ public class EntityCreationService {
}
private static Set<Engine> getNerEngines(NerEntities nerEntities) {
return getNerEngines(nerEntities.getNerEntityList()
.toArray(new NerEntities.NerEntity[0]));
}
private static Set<Engine> getNerEngines(NerEntities.NerEntity... nerEntities) {
return Arrays.stream(nerEntities)
.map(NerEntities.NerEntity::engine)
.map(NerEntities::mapToPrimaryEngine)
.collect(Collectors.toSet());
}
private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
entityToDuplicate.addTextRange(newTextRange);

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java
View File

@ -123,7 +123,7 @@ public class ComponentDroolsExecutionService {
logger.error(e, "Exception during rule execution");
kieSession.dispose();
if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.COMPONENT);
}
throw new RuntimeException(e);
} catch (InterruptedException e) {
@ -131,7 +131,7 @@ public class ComponentDroolsExecutionService {
kieSession.dispose();
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.COMPONENT);
}
List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
View File

@ -156,7 +156,7 @@ public class EntityDroolsExecutionService {
logger.error(e, "Exception during rule execution");
kieSession.dispose();
if (e.getCause() instanceof TimeoutException) {
throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY);
}
throw new RuntimeException(e);
} catch (InterruptedException e) {
@ -164,7 +164,7 @@ public class EntityDroolsExecutionService {
kieSession.dispose();
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY);
}
List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -6,6 +6,7 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@ -31,7 +32,6 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.llm.service.LlmNerEntities;
@ -137,13 +137,13 @@ public class RedactionStorageService {
}
@SneakyThrows
public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) {
componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog);
}
@SneakyThrows
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
@ -373,7 +373,12 @@ public class RedactionStorageService {
public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class);
String objectId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES);
if (storageService.objectExists(TenantContext.getTenantId(), objectId)) {
return storageService.readJSONObject(TenantContext.getTenantId(), objectId, LlmNerEntities.class);
} else {
return LlmNerEntities.builder().entities(new ArrayList<>()).build();
}
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java
View File

@ -22,6 +22,14 @@ public class DroolsTimeoutException extends RuntimeException {
}
public DroolsTimeoutException(String message, Throwable cause, boolean reported, RuleFileType ruleFileType) {
super(message, cause);
this.reported = reported;
this.ruleFileType = ruleFileType;
}
public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) {
super(DROOLS_TIMEOUT_MESSAGE);

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -261,6 +261,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
}
@Disabled
@Test
public void testDoseMortalityExtraction() {

125
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
import static com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
@ -45,11 +46,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@ -61,6 +64,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
@ -2164,7 +2168,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
String assessment = "assessment";
dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment);
reanlysisVersions.put(assessment, 1L);
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L);
analyzeService.reanalyze(request);
@ -2191,6 +2194,126 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
}
@Test
@SneakyThrows
@Order(3)
public void testLocalRemovalOfDictEntry() {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
String pdfFile = "files/new/test_file.pdf";
String manualDictAddId = UUID.randomUUID().toString();
String manualAddId = UUID.randomUUID().toString();
String valueToAdd = "Crandu Seku Laku Meku";
// positions=[Rectangle(topLeft=Point(x=56.8, y=527.264), width=120.96, height=15.408, page=1)],
List<Rectangle> fullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(527.264f).width(120.96f).height(15.408f).page(1).build());
ManualRedactionEntry manualDictRedactionEntry = new ManualRedactionEntry();
manualDictRedactionEntry.setAnnotationId(manualDictAddId);
manualDictRedactionEntry.setFileId("fileId");
manualDictRedactionEntry.setUser("test");
manualDictRedactionEntry.setType("CBI_author");
manualDictRedactionEntry.setRectangle(false);
manualDictRedactionEntry.setAddToDictionary(true);
manualDictRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY);
manualDictRedactionEntry.setRequestDate(OffsetDateTime.now());
manualDictRedactionEntry.setValue(valueToAdd);
manualDictRedactionEntry.setReason("Dictionary Request");
manualDictRedactionEntry.setPositions(fullPositions);
var idUsedForRemoval = "4d92d86e7d70ab9bb5c0d554bfa3c7f0";
var idRemoval = getIdRemoval(idUsedForRemoval);
AnalyzeRequest request = uploadFileToStorage(pdfFile);
Set<ManualRedactionEntry> entriesToAdd = Set.of(manualDictRedactionEntry);
Set<IdRemoval> entriesToRemove = Set.of(idRemoval);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).idsToRemove(entriesToRemove).build());
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
request.setAnalysisNumber(0);
mockDictionaryCalls(1L);
dossierDictionary.get(DICTIONARY_AUTHOR).add(valueToAdd);
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L);
reanlysisVersions.put(valueToAdd, 0L);
analyzeService.analyze(request);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(entityLog.getEntityLogEntry().size(), 4);
EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval))
.findFirst()
.get();
assertEquals(entityLogEntry1.getState(), EntryState.IGNORED);
assertEquals(entityLogEntry1.getMatchedRule(), "CBI.0.3");
assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL));
//-----------force local
var processedDate = OffsetDateTime.now();
manualDictRedactionEntry.setProcessedDate(processedDate);
idRemoval.setProcessedDate(processedDate);
//[Rectangle(topLeft=Point(x=56.8, y=528.9), width=120.96001, height=12.642, page=1)]
List<Rectangle> localfullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(528.9f).width(120.96f).height(12.642f).page(1).build());
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setUser("test");
manualRedactionEntry.setType("CBI_author");
manualRedactionEntry.setRectangle(false);
manualRedactionEntry.setAddToDictionary(false);
manualRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY);
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
manualRedactionEntry.setValue(valueToAdd);
manualRedactionEntry.setReason("Author found, removed by manual override");
manualRedactionEntry.setSection("Header: This is my test");
manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
manualRedactionEntry.setTextBefore("Lorem My Ipsum ");
manualRedactionEntry.setTextAfter("Crandu Seku Laku");
manualRedactionEntry.setPositions(localfullPositions);
ManualForceRedaction forceRequest = new ManualForceRedaction();
forceRequest.setAnnotationId(manualAddId);
forceRequest.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
forceRequest.setUser("test");
forceRequest.setRequestDate(OffsetDateTime.now());
Set<ManualForceRedaction> forceRedactions = Set.of(forceRequest);
request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualDictRedactionEntry, manualRedactionEntry))
.idsToRemove(entriesToRemove)
.forceRedactions(forceRedactions)
.build());
request.setAnalysisNumber(2);
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(entityLog.getEntityLogEntry().size(), 5);
entityLogEntry1 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval))
.findFirst()
.get();
var entityLogEntry2 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get();
assertEquals(entityLogEntry1.getState(), EntryState.REMOVED);
assertEquals(entityLogEntry1.getMatchedRule(), "X.11.1");
assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL));
assertEquals(entityLogEntry2.getState(), EntryState.APPLIED);
assertEquals(entityLogEntry2.getMatchedRule(), "MAN.5.0");
dossierDictionary.get(DICTIONARY_AUTHOR).remove(valueToAdd);
}
@Test
@SneakyThrows
public void testDocumentDataFallback() {

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java
View File

@ -30,8 +30,8 @@ public class TextEntityTest {
entity.apply("MAN.3.0", "");
entity.apply("MAN.3.3", "");
entity.skip("CBI.13.2", "");
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("MAN.3.0");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(3);
}

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf Normal file
View File

Binary file not shown.

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta

@ -1 +1 @@
Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421
Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7