Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Revert "RED-7327 - Change to group annotation"

Browse Source
This commit is contained in:
Maverick Studer 2024-08-29 10:18:48 +02:00
parent ea7137b242
commit b9240dec68
23 changed files with 57 additions and 692 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -4,7 +4,7 @@ plugins {
} }
description = "redaction-service-api-v1" description = "redaction-service-api-v1"
val persistenceServiceVersion = "2.532.0" val persistenceServiceVersion = "2.536.0"
dependencies { dependencies {
implementation("org.springframework:spring-web:6.0.12") implementation("org.springframework:spring-web:6.0.12")

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.161.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final" val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.532.0" val persistenceServiceVersion = "2.536.0"
val llmServiceVersion = "1.11.0" val llmServiceVersion = "1.11.0"
val springBootStarterVersion = "3.1.5" val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4" val springCloudVersion = "4.0.4"

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java
View File

@ -1,10 +0,0 @@
package com.iqser.red.service.redaction.v1.server.client;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.internal.resources.GroupAnnotationResource;
@FeignClient(name = "GroupAnnotationResource", url = "${persistence-service.url}")
public interface GroupRedactionClient extends GroupAnnotationResource {
}

47
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model;
import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE; import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.PriorityQueue; import java.util.PriorityQueue;
@ -15,13 +14,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -51,7 +48,6 @@ public class PrecursorEntity implements IEntity {
boolean isDossierDictionaryEntry; boolean isDossierDictionaryEntry;
boolean rectangle; boolean rectangle;
Set<Engine> engines; Set<Engine> engines;
String groupId;
@Builder.Default @Builder.Default
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>(); PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
@ -153,49 +149,6 @@ public class PrecursorEntity implements IEntity {
} }
public static List<PrecursorEntity> fromAreaGroupAnnotation(AreaGroupAnnotation areaGroupAnnotation) {
List<PrecursorEntity> precursorEntities = new ArrayList<>();
areaGroupAnnotation.getPageRanges()
.forEach(pageRange -> {
for (int pageStart = pageRange.getStartPage(); pageStart <= pageRange.getEndPage(); pageStart++) {
boolean isHint = areaGroupAnnotation.getEntryType() == EntryType.HINT;
String ruleIdentifier = isHint ? "GRP.5.0" : "GRP.5.1";
String reason = isHint ? "group hint is skipped by default" : "group area is applied by default";
String legalBasis = isHint ? "" : areaGroupAnnotation.getLegalBasis();
MatchedRule.MatchedRuleBuilder matchedRule = MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason);
if (!isHint) {
matchedRule.legalBasis(legalBasis);
matchedRule.applied(true);
}
precursorEntities.add(PrecursorEntity.builder()
.id(UUID.randomUUID().toString())
.value(areaGroupAnnotation.getValue())
.entityPosition(List.of(RectangleWithPage.fromPositionOnPage(pageStart, areaGroupAnnotation.getPositionOnPage())))
.ruleIdentifier(ruleIdentifier)
.matchedRuleList(new PriorityQueue<>(List.of(matchedRule.build())))
.reason(reason)
.legalBasis(legalBasis)
.section(areaGroupAnnotation.getSection())
.type(areaGroupAnnotation.getTypeId())
.entityType(isHint ? EntityType.HINT : EntityType.ENTITY)
.applied(!isHint)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
.rectangle(true)
.section(areaGroupAnnotation.getSection())
.engines(Set.of(Engine.GROUP))
.groupId(areaGroupAnnotation.getGroupId())
.manualOverwrite(new ManualChangeOverwrite(isHint ? EntityType.HINT : EntityType.ENTITY))
.build());
}
});
return precursorEntities;
}
public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions() List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.model;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
@ -20,12 +19,6 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
} }
public static RectangleWithPage fromPositionOnPage(int pageNumber, PositionOnPage positionOnPage) {
return new RectangleWithPage(pageNumber, toRectangle2D(positionOnPage));
}
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) { public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle)); return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
@ -49,10 +42,4 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()); return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
} }
private static Rectangle2D toRectangle2D(PositionOnPage positionOnPage) {
return new Rectangle2D.Float(positionOnPage.getX(), positionOnPage.getY(), positionOnPage.getWidth(), positionOnPage.getHeight());
}
} }

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java
View File

@ -19,7 +19,6 @@ import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Setter;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@Builder @Builder

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -29,9 +29,7 @@ public final class MatchedRule implements Comparable<MatchedRule> {
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP"); public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT"); public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN"); private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
public static final RuleType GROUP_TYPE = RuleType.fromString("GRP");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE, MANUAL_TYPE, GROUP_TYPE);
RuleIdentifier ruleIdentifier; RuleIdentifier ruleIdentifier;
@Builder.Default @Builder.Default

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -53,7 +53,6 @@ public class TextEntity implements IEntity {
boolean dictionaryEntry; boolean dictionaryEntry;
boolean dossierDictionaryEntry; boolean dossierDictionaryEntry;
String groupId;
@Builder.Default @Builder.Default
Set<Engine> engines = new HashSet<>(); Set<Engine> engines = new HashSet<>();

152
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -6,11 +6,9 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
@ -20,12 +18,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.NerEntities;
@ -36,7 +29,6 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.AreaGroupAnnotationService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
@ -45,7 +37,6 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ -65,9 +56,7 @@ public class AnalysisPreparationService {
ImportedRedactionEntryService importedRedactionEntryService; ImportedRedactionEntryService importedRedactionEntryService;
DictionaryService dictionaryService; DictionaryService dictionaryService;
SectionFinderService sectionFinderService; SectionFinderService sectionFinderService;
GroupRedactionClient groupRedactionClient;
TaskExecutor taskExecutor; TaskExecutor taskExecutor;
AreaGroupAnnotationService areaGroupAnnotationService;
public AnalysisPreparationService(KieContainerCreationService kieContainerCreationService, public AnalysisPreparationService(KieContainerCreationService kieContainerCreationService,
@ -78,9 +67,7 @@ public class AnalysisPreparationService {
ImportedRedactionEntryService importedRedactionEntryService, ImportedRedactionEntryService importedRedactionEntryService,
DictionaryService dictionaryService, DictionaryService dictionaryService,
SectionFinderService sectionFinderService, SectionFinderService sectionFinderService,
GroupRedactionClient groupRedactionClient, @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) {
@Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor,
AreaGroupAnnotationService areaGroupAnnotationService) {
this.kieContainerCreationService = kieContainerCreationService; this.kieContainerCreationService = kieContainerCreationService;
this.observedStorageService = observedStorageService; this.observedStorageService = observedStorageService;
@ -90,9 +77,7 @@ public class AnalysisPreparationService {
this.importedRedactionEntryService = importedRedactionEntryService; this.importedRedactionEntryService = importedRedactionEntryService;
this.dictionaryService = dictionaryService; this.dictionaryService = dictionaryService;
this.sectionFinderService = sectionFinderService; this.sectionFinderService = sectionFinderService;
this.groupRedactionClient = groupRedactionClient;
this.taskExecutor = taskExecutor; this.taskExecutor = taskExecutor;
this.areaGroupAnnotationService = areaGroupAnnotationService;
} }
@ -109,10 +94,6 @@ public class AnalysisPreparationService {
CompletableFuture<NerEntities> nerEntitiesFuture = documentFuture.thenApplyAsync((document) -> getNerEntities(analyzeRequest, document), taskExecutor); CompletableFuture<NerEntities> nerEntitiesFuture = documentFuture.thenApplyAsync((document) -> getNerEntities(analyzeRequest, document), taskExecutor);
CompletableFuture<GroupAnnotationInternalResponse> groupAnnotationInternalResponseCompletableFuture = CompletableFuture.supplyAsync(() -> getGroupAnnotations(analyzeRequest,
false),
taskExecutor);
CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join(); CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join();
Dictionary dictionary = getDictionary(analyzeRequest); Dictionary dictionary = getDictionary(analyzeRequest);
@ -126,9 +107,6 @@ public class AnalysisPreparationService {
List<PrecursorEntity> notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); List<PrecursorEntity> notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
List<PrecursorEntity> areaGroupRedactions = areaGroupAnnotationService.createAreaGroupAnnotations(groupAnnotationInternalResponseCompletableFuture.get()
.getAreaGroupAnnotations());
return new AnalysisData(kieWrapperEntityRulesFuture.get(), return new AnalysisData(kieWrapperEntityRulesFuture.get(),
kieWrapperComponentRulesFuture.get(), kieWrapperComponentRulesFuture.get(),
document, document,
@ -136,9 +114,7 @@ public class AnalysisPreparationService {
dictionary, dictionary,
notFoundManualRedactionEntries, notFoundManualRedactionEntries,
notFoundImportedEntries, notFoundImportedEntries,
nerEntitiesFuture.get(), nerEntitiesFuture.get());
groupAnnotationInternalResponseCompletableFuture.get(),
areaGroupRedactions);
} }
@ -160,35 +136,11 @@ public class AnalysisPreparationService {
CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor); CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
CompletableFuture<GroupAnnotationInternalResponse> internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupAnnotations(analyzeRequest, true), taskExecutor); CompletableFuture<SectionsToReanalyzeData> incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenApplyAsync((importedRedactions) -> {
DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(analyzeRequest, reanalysisSetupData);
return getDictionaryIncrementAndSectionsToReanalyze(analyzeRequest, dictionaryIncrement, reanalysisSetupData, importedRedactions);
CompletableFuture<SectionsToReanalyzeData> incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenCombineAsync(internalResponseFuture, }, taskExecutor);
(importedRedactions, groupRedactions) -> {
List<AreaGroupAnnotation> areaGroupRedactions = groupRedactions.getAreaGroupAnnotations();
List<TextGroupAnnotation> textGroupRedactions = groupRedactions.getTextGroupAnnotations();
List<GroupAnnotation> groupAnnotations = Stream.concat(
Optional.ofNullable(
areaGroupRedactions)
.orElseGet(List::of)
.stream(),
Optional.ofNullable(
textGroupRedactions)
.orElseGet(List::of)
.stream())
.toList();
DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(
analyzeRequest,
reanalysisSetupData);
return getDictionaryIncrementAndSectionsToReanalyze(
analyzeRequest,
dictionaryIncrement,
reanalysisSetupData,
importedRedactions,
groupAnnotations);
},
taskExecutor);
CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor); CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
@ -196,27 +148,19 @@ public class AnalysisPreparationService {
CompletableFuture.allOf(importedRedactionsFuture, incrementAndSectionsToReanalyzeFuture, kieWrapperComponentRulesFuture, kieWrapperEntityRulesFuture).join(); CompletableFuture.allOf(importedRedactionsFuture, incrementAndSectionsToReanalyzeFuture, kieWrapperComponentRulesFuture, kieWrapperEntityRulesFuture).join();
List<AreaGroupAnnotation> areaGroupAnnotations = internalResponseFuture.get().getAreaGroupAnnotations()
.stream()
.filter(UnprocessedUtils::isUnprocessedGroupAnnotation)
.toList();
return new ReanalysisInitialProcessingData(importedRedactionsFuture.get(), return new ReanalysisInitialProcessingData(importedRedactionsFuture.get(),
incrementAndSectionsToReanalyzeFuture.get().dictionaryIncrement(), incrementAndSectionsToReanalyzeFuture.get().dictionaryIncrement(),
incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyseIds(), incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyseIds(),
incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyze(), incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyze(),
kieWrapperComponentRulesFuture.get(), kieWrapperComponentRulesFuture.get(),
kieWrapperEntityRulesFuture.get(), kieWrapperEntityRulesFuture.get());
internalResponseFuture.get(),
areaGroupAnnotations);
} }
@SneakyThrows @SneakyThrows
public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest, public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest,
ReanalysisSetupData reanalysisSetupData, ReanalysisSetupData reanalysisSetupData,
ReanalysisInitialProcessingData reanalysisInitialProcessingData, ReanalysisInitialProcessingData reanalysisInitialProcessingData) {
GroupAnnotationInternalResponse groupAnnotationInternalResponse) {
CompletableFuture<NerEntities> nerEntitiesFuture = CompletableFuture.supplyAsync(() -> getNerEntitiesFiltered(analyzeRequest, CompletableFuture<NerEntities> nerEntitiesFuture = CompletableFuture.supplyAsync(() -> getNerEntitiesFiltered(analyzeRequest,
reanalysisSetupData.document, reanalysisSetupData.document,
@ -225,30 +169,16 @@ public class AnalysisPreparationService {
CompletableFuture<DictionaryAndNotFoundEntries> dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> { CompletableFuture<DictionaryAndNotFoundEntries> dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> {
Dictionary dictionary = getDictionary(analyzeRequest); Dictionary dictionary = getDictionary(analyzeRequest);
NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions());
reanalysisSetupData.document(), return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries());
reanalysisInitialProcessingData.importedRedactions(),
groupAnnotationInternalResponse.getAreaGroupAnnotations()
.stream()
.filter(gR -> gR.getSoftDeletedTime() == null)
.toList());
return new DictionaryAndNotFoundEntries(dictionary,
notFoundEntries.notFoundManualRedactionEntries(),
notFoundEntries.notFoundImportedEntries(),
notFoundEntries.areaGroupRedactions());
}, taskExecutor); }, taskExecutor);
CompletableFuture<GroupAnnotationInternalResponse> groupAnnotationInternalResponseCompletableFuture = CompletableFuture.supplyAsync(() -> getGroupAnnotations(analyzeRequest, CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture).join();
false),
taskExecutor);
CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture, groupAnnotationInternalResponseCompletableFuture).join();
return new ReanalysisFinalProcessingData(nerEntitiesFuture.get(), return new ReanalysisFinalProcessingData(nerEntitiesFuture.get(),
dictionaryAndNotFoundEntriesCompletableFuture.get().dictionary(), dictionaryAndNotFoundEntriesCompletableFuture.get().dictionary(),
dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundManualRedactionEntries(), dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundManualRedactionEntries(),
dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries(), dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries());
dictionaryAndNotFoundEntriesCompletableFuture.get().areaGroupRedactions());
} }
@ -272,16 +202,6 @@ public class AnalysisPreparationService {
} }
public GroupAnnotationInternalResponse getGroupAnnotations(AnalyzeRequest analyzeRequest, boolean includeDeleted) {
GroupAnnotationInternalResponse groupRedactionResponse = groupRedactionClient.getGroupAnnotations(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
includeDeleted);
log.info("Loaded group redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
return groupRedactionResponse;
}
public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) { public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
@ -315,17 +235,13 @@ public class AnalysisPreparationService {
} }
private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) {
Document document,
ImportedRedactions importedRedactions,
List<AreaGroupAnnotation> groupAnnotations) {
var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document, document,
analyzeRequest.getDossierTemplateId()); analyzeRequest.getDossierTemplateId());
var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
var areaGroupRedactions = areaGroupAnnotationService.createAreaGroupAnnotations(groupAnnotations); return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries);
return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries, areaGroupRedactions);
} }
@ -376,8 +292,7 @@ public class AnalysisPreparationService {
private SectionsToReanalyzeData getDictionaryIncrementAndSectionsToReanalyze(AnalyzeRequest analyzeRequest, private SectionsToReanalyzeData getDictionaryIncrementAndSectionsToReanalyze(AnalyzeRequest analyzeRequest,
DictionaryIncrement dictionaryIncrement, DictionaryIncrement dictionaryIncrement,
ReanalysisSetupData reanalysisSetupData, ReanalysisSetupData reanalysisSetupData,
ImportedRedactions importedRedactions, ImportedRedactions importedRedactions) {
List<GroupAnnotation> groupAnnotations) {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@ -388,8 +303,7 @@ public class AnalysisPreparationService {
reanalysisSetupData.document(), reanalysisSetupData.document(),
dictionaryIncrement, dictionaryIncrement,
importedRedactions, importedRedactions,
relevantManuallyModifiedAnnotationIds, relevantManuallyModifiedAnnotationIds));
groupAnnotations));
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(reanalysisSetupData.document(), sectionsToReanalyseIds); List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(reanalysisSetupData.document(), sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -460,31 +374,21 @@ public class AnalysisPreparationService {
Document document, Document document,
DictionaryIncrement dictionaryIncrement, DictionaryIncrement dictionaryIncrement,
ImportedRedactions importedRedactions, ImportedRedactions importedRedactions,
Set<String> relevantManuallyModifiedAnnotationIds, Set<String> relevantManuallyModifiedAnnotationIds) {
List<GroupAnnotation> groupAnnotations) {
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds);
document,
analyzeRequest,
importedRedactions,
relevantManuallyModifiedAnnotationIds,
groupAnnotations);
} }
private record DictionaryAndNotFoundEntries( private record DictionaryAndNotFoundEntries(Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries) {
Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries, List<PrecursorEntity> areaGroupRedactions
) {
} }
private record NotFoundEntries(List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries, List<PrecursorEntity> areaGroupRedactions) { private record NotFoundEntries(List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries) {
} }
private record SectionsToReanalyzeData( private record SectionsToReanalyzeData(DictionaryIncrement dictionaryIncrement, Set<Integer> sectionsToReanalyseIds, List<SemanticNode> sectionsToReanalyze) {
DictionaryIncrement dictionaryIncrement, Set<Integer> sectionsToReanalyseIds, List<SemanticNode> sectionsToReanalyze
) {
} }
@ -496,9 +400,7 @@ public class AnalysisPreparationService {
Dictionary dictionary, Dictionary dictionary,
List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundImportedEntries, List<PrecursorEntity> notFoundImportedEntries,
NerEntities nerEntities, NerEntities nerEntities
GroupAnnotationInternalResponse groupRedactions,
List<PrecursorEntity> areaGroupRedactions
) { ) {
} }
@ -515,19 +417,13 @@ public class AnalysisPreparationService {
Set<Integer> sectionsToReanalyseIds, Set<Integer> sectionsToReanalyseIds,
List<SemanticNode> sectionsToReAnalyse, List<SemanticNode> sectionsToReAnalyse,
KieWrapper kieWrapperComponentRules, KieWrapper kieWrapperComponentRules,
KieWrapper kieWrapperEntityRules, KieWrapper kieWrapperEntityRules
GroupAnnotationInternalResponse groupAnnotations,
List<AreaGroupAnnotation> unprocessedAreaGroupAnnotations
) { ) {
} }
public record ReanalysisFinalProcessingData( public record ReanalysisFinalProcessingData(
NerEntities nerEntities, NerEntities nerEntities, Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries
Dictionary dictionary,
List<PrecursorEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundImportedEntries,
List<PrecursorEntity> areaGroupAnnotations
) { ) {
} }

78
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -6,9 +6,9 @@ import static com.iqser.red.service.redaction.v1.server.service.AnalysisPreparat
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
@ -24,11 +24,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.logger.Context;
@ -67,7 +64,6 @@ public class AnalyzeService {
FunctionTimerValues redactmanagerAnalyzePagewiseValues; FunctionTimerValues redactmanagerAnalyzePagewiseValues;
ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE; ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;
AnalysisPreparationService analysisPreparationService; AnalysisPreparationService analysisPreparationService;
GroupSearchService groupSearchService;
@Timed("redactmanager_reanalyze") @Timed("redactmanager_reanalyze")
@ -92,7 +88,7 @@ public class AnalyzeService {
ReanalysisInitialProcessingData initialProcessingData = analysisPreparationService.getReanalysisInitialProcessingData(analyzeRequest, setupData); ReanalysisInitialProcessingData initialProcessingData = analysisPreparationService.getReanalysisInitialProcessingData(analyzeRequest, setupData);
if (initialProcessingData.sectionsToReAnalyse().isEmpty() && initialProcessingData.groupAnnotations().getAreaGroupAnnotations().isEmpty()) { if (initialProcessingData.sectionsToReAnalyse().isEmpty()) {
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(setupData.entityLog(), EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(setupData.entityLog(),
initialProcessingData.dictionaryIncrement().getDictionaryVersion(), initialProcessingData.dictionaryIncrement().getDictionaryVersion(),
@ -100,8 +96,6 @@ public class AnalyzeService {
new ArrayList<>(), new ArrayList<>(),
new ArrayList<>()); new ArrayList<>());
Set<GroupAnnotation> groupAnnotations = getGroupAnnotations(initialProcessingData.groupAnnotations());
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
initialProcessingData.kieWrapperComponentRules(), initialProcessingData.kieWrapperComponentRules(),
@ -110,28 +104,17 @@ public class AnalyzeService {
setupData.document().getNumberOfPages(), setupData.document().getNumberOfPages(),
true, true,
new HashSet<>(), new HashSet<>(),
groupAnnotations,
context); context);
} }
context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion()); context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion());
ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, setupData, initialProcessingData);
setupData,
initialProcessingData,
initialProcessingData.groupAnnotations());
if (!initialProcessingData.sectionsToReAnalyse().isEmpty()) { dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse());
dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
groupSearchService.addGroupEntities(initialProcessingData.groupAnnotations().getTextGroupAnnotations(), initialProcessingData.sectionsToReAnalyse()); var notFoundManualOrImportedEntries = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), finalProcessingData.notFoundImportedEntries())
log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
var notFoundManualOrImportedEntriesOrAreaGroupRedactions = Stream.of(finalProcessingData.notFoundManualRedactionEntries(),
finalProcessingData.notFoundImportedEntries(),
finalProcessingData.areaGroupAnnotations())
.flatMap(Collection::stream) .flatMap(Collection::stream)
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -149,14 +132,12 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
setupData.document(), setupData.document(),
setupData.entityLog(), setupData.entityLog(),
notFoundManualOrImportedEntriesOrAreaGroupRedactions, notFoundManualOrImportedEntries,
initialProcessingData.sectionsToReanalyseIds(), initialProcessingData.sectionsToReanalyseIds(),
finalProcessingData.dictionary().getVersion()); finalProcessingData.dictionary().getVersion());
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries());
Set<GroupAnnotation> groupAnnotations = getGroupAnnotations(initialProcessingData.groupAnnotations());
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
initialProcessingData.kieWrapperComponentRules(), initialProcessingData.kieWrapperComponentRules(),
@ -165,7 +146,6 @@ public class AnalyzeService {
setupData.document().getNumberOfPages(), setupData.document().getNumberOfPages(),
true, true,
new HashSet<>(allFileAttributes), new HashSet<>(allFileAttributes),
groupAnnotations,
context); context);
} }
@ -186,18 +166,13 @@ public class AnalyzeService {
analyzeRequest.getAnalysisNumber(), analyzeRequest.getAnalysisNumber(),
TenantContext.getTenantId()); TenantContext.getTenantId());
var notFoundManualOrImportedEntriesOrGroupAreaRedaction = Stream.of(analysisData.notFoundManualRedactionEntries(), var notFoundManualOrImportedEntries = Stream.of(analysisData.notFoundManualRedactionEntries(), analysisData.notFoundImportedEntries())
analysisData.notFoundImportedEntries(),
analysisData.areaGroupRedactions())
.flatMap(Collection::stream) .flatMap(Collection::stream)
.collect(Collectors.toList()); .collect(Collectors.toList());
dictionarySearchService.addDictionaryEntities(analysisData.dictionary(), analysisData.document()); dictionarySearchService.addDictionaryEntities(analysisData.dictionary(), analysisData.document());
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
groupSearchService.addGroupEntities(analysisData.groupRedactions().getTextGroupAnnotations(), analysisData.document());
log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// we could add the imported redactions similar to the manual redactions here as well for additional processing // we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(analysisData.kieWrapperEntityRules().container(), List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(analysisData.kieWrapperEntityRules().container(),
analysisData.document(), analysisData.document(),
@ -210,14 +185,12 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
analysisData.document(), analysisData.document(),
notFoundManualOrImportedEntriesOrGroupAreaRedaction, notFoundManualOrImportedEntries,
analysisData.dictionary().getVersion(), analysisData.dictionary().getVersion(),
analysisData.kieWrapperEntityRules().rulesVersion()); analysisData.kieWrapperEntityRules().rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
Set<GroupAnnotation> groupAnnotations = getGroupAnnotations(analysisData.groupRedactions());
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
analysisData.kieWrapperComponentRules(), analysisData.kieWrapperComponentRules(),
@ -226,11 +199,9 @@ public class AnalyzeService {
analysisData.document().getNumberOfPages(), analysisData.document().getNumberOfPages(),
false, false,
new HashSet<>(allFileAttributes), new HashSet<>(allFileAttributes),
groupAnnotations,
context); context);
} }
@Timed("redactmanager_analyzeImportedRedactionsOnly") @Timed("redactmanager_analyzeImportedRedactionsOnly")
@Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly") @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly")
public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) { public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) {
@ -266,17 +237,7 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
Set<GroupAnnotation> groupAnnotations = getGroupAnnotations(analysisData.groupRedactions()); return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), entityLogChanges, analysisData.document(), analysisData.document().getNumberOfPages(), false, new HashSet<>(),
return finalizeAnalysis(analyzeRequest,
startTime,
analysisData.kieWrapperComponentRules(),
entityLogChanges,
analysisData.document(),
analysisData.document().getNumberOfPages(),
false,
new HashSet<>(),
groupAnnotations,
context); context);
} }
@ -289,7 +250,6 @@ public class AnalyzeService {
int numberOfPages, int numberOfPages,
boolean isReanalysis, boolean isReanalysis,
Set<FileAttribute> addedFileAttributes, Set<FileAttribute> addedFileAttributes,
Set<GroupAnnotation> groupAnnotations,
Context context) { Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog(); EntityLog entityLog = entityLogChanges.getEntityLog();
@ -335,7 +295,6 @@ public class AnalyzeService {
.manualRedactions(analyzeRequest.getManualRedactions()) .manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes) .addedFileAttributes(addedFileAttributes)
.usedComponentMappings(analyzeRequest.getComponentMappings()) .usedComponentMappings(analyzeRequest.getComponentMappings())
.groupAnnotations(groupAnnotations)
.build(); .build();
} }
@ -370,19 +329,4 @@ public class AnalyzeService {
log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} }
private Set<GroupAnnotation> getGroupAnnotations(GroupAnnotationInternalResponse groupAnnotationInternalResponse) {
List<AreaGroupAnnotation> areaGroupRedactions = groupAnnotationInternalResponse.getAreaGroupAnnotations();
List<TextGroupAnnotation> textGroupRedactions = groupAnnotationInternalResponse.getTextGroupAnnotations();
return Stream.concat(Optional.ofNullable(areaGroupRedactions)
.orElseGet(List::of)
.stream(),
Optional.ofNullable(textGroupRedactions)
.orElseGet(List::of)
.stream())
.collect(Collectors.toSet());
}
} }

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -15,7 +15,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
@ -45,25 +44,10 @@ public class EntityChangeLogService {
List<EntityLogEntry> toInsert = new ArrayList<>(); List<EntityLogEntry> toInsert = new ArrayList<>();
List<EntityLogEntry> toUpdate = new ArrayList<>(); List<EntityLogEntry> toUpdate = new ArrayList<>();
List<EntityLogEntry> existingAreaGroupEntries = new ArrayList<>();
for (EntityLogEntry entityLogEntry : newEntityLogEntries) { for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream() Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getId().equals(entityLogEntry.getId())) .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.findAny(); .findAny();
Optional<EntityLogEntry> optionalPreviousAreaGroupEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getGroupId() != null && entry.getGroupId().equals(entityLogEntry.getGroupId()))
.filter(entry -> entry.getEntryType().equals(EntryType.AREA))
.filter(entry -> entry.getPositions().equals(entityLogEntry.getPositions()))
.findAny();
// Skip group area entities on the same position
if (optionalPreviousAreaGroupEntity.isPresent()) {
existingAreaGroupEntries.add(optionalPreviousAreaGroupEntity.get());
continue;
}
if (optionalPreviousEntity.isEmpty()) { if (optionalPreviousEntity.isEmpty()) {
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())); entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()));
toInsert.add(entityLogEntry); toInsert.add(entityLogEntry);
@ -81,21 +65,19 @@ public class EntityChangeLogService {
} }
} }
toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, existingAreaGroupEntries, analysisNumber, now)); toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now));
return new EntryChanges(toInsert, toUpdate); return new EntryChanges(toInsert, toUpdate);
} }
private List<EntityLogEntry> addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries, private List<EntityLogEntry> addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
List<EntityLogEntry> newEntityLogEntries, List<EntityLogEntry> existingAreaGroupEntries, int analysisNumber, List<EntityLogEntry> newEntityLogEntries,
int analysisNumber,
OffsetDateTime now) { OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream() Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId) .map(EntityLogEntry::getId)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
existingIds.addAll(existingAreaGroupEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet()));
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream() List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !existingIds.contains(entry.getId())) .filter(entry -> !existingIds.contains(entry.getId()))
.toList(); .toList();

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -238,7 +238,7 @@ public class EntityLogCreatorService {
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
EntityLogEntry entityLogEntry = EntityLogEntry.builder() return EntityLogEntry.builder()
.id(precursorEntity.getId()) .id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis()) .legalBasis(precursorEntity.legalBasis())
@ -267,12 +267,9 @@ public class EntityLogCreatorService {
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
.imported(false) .imported(false)
.reference(Collections.emptySet()) .reference(Collections.emptySet())
.manualChanges(precursorEntity.getGroupId() == null ? ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges) : Collections.emptyList()) .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
.groupId(precursorEntity.getGroupId())
.paragraphPageIdx(-1) .paragraphPageIdx(-1)
.build(); .build();
return entityLogEntry;
} }
@ -321,7 +318,6 @@ public class EntityLogCreatorService {
.state(buildEntryState(entity)) .state(buildEntryState(entity))
.entryType(entryType) .entryType(entryType)
.paragraphPageIdx(determinePageParagraphIndex(entity, entryType)) .paragraphPageIdx(determinePageParagraphIndex(entity, entryType))
.groupId(entity.getGroupId())
.build(); .build();
} }

92
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java
View File

@ -1,92 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class GroupSearchService {
EntityEnrichmentService entityEnrichmentService;
public void addGroupEntities(List<TextGroupAnnotation> groupAnnotations, List<SemanticNode> nodes) {
nodes.forEach(node -> addGroupEntities(groupAnnotations, node));
}
public void addGroupEntities(List<TextGroupAnnotation> groupAnnotations, SemanticNode node) {
groupAnnotations.stream()
.filter(groupRedaction -> groupRedaction.getSoftDeletedTime() == null)
.forEach(groupRedaction -> {
SearchImplementation searchImplementation = new SearchImplementation(groupRedaction.getValue(), false);
bySearchImplementationAsGroup(searchImplementation,
groupRedaction.getTypeId(),
calculateEntityType(groupRedaction.getEntryType()),
groupRedaction.getGroupId(),
groupRedaction.getLegalBasis(),
node);
});
}
public void bySearchImplementationAsGroup(SearchImplementation searchImplementation, String type, EntityType entityType, String groupId, String legalBasis, SemanticNode node) {
Set<Engine> engines = Set.of(Engine.GROUP);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
.ifPresent(entity -> {
if (entity.getEntityType().equals(EntityType.HINT)) {
entity.skip("GRP.0.1", "group hint is skipped by default");
} else {
entity.apply("GRP.0.0", "group entries are applied by default", legalBasis);
}
entity.setGroupId(groupId);
}));
}
private EntityType calculateEntityType(EntryType entryType) {
switch (entryType) {
case FALSE_RECOMMENDATION -> {
return EntityType.FALSE_RECOMMENDATION;
}
case RECOMMENDATION -> {
return EntityType.RECOMMENDATION;
}
case HINT -> {
return EntityType.HINT;
}
case FALSE_POSITIVE -> {
return EntityType.FALSE_POSITIVE;
}
default -> {
return EntityType.ENTITY;
}
}
}
}

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupAnnotationService.java
View File

@ -1,28 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.ArrayList;
import java.util.List;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class AreaGroupAnnotationService {
public List<PrecursorEntity> createAreaGroupAnnotations(List<AreaGroupAnnotation> groupAnnotations) {
List<PrecursorEntity> precursorEntities = new ArrayList<>();
if (groupAnnotations != null) {
groupAnnotations.forEach(groupAnnotation -> precursorEntities.addAll(PrecursorEntity.fromAreaGroupAnnotation(groupAnnotation)));
}
return precursorEntities;
}
}

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -11,6 +11,7 @@ import java.util.stream.Stream;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@ -22,14 +23,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationType;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel; import lombok.AccessLevel;
@ -48,29 +46,20 @@ public class SectionFinderService {
Document document, Document document,
AnalyzeRequest analyzeRequest, AnalyzeRequest analyzeRequest,
ImportedRedactions importedRedactions, ImportedRedactions importedRedactions,
Set<String> relevantManuallyModifiedAnnotationIds, Set<String> relevantManuallyModifiedAnnotationIds) {
List<GroupAnnotation> groupAnnotations) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
Set<Integer> sectionsToReanalyse = new HashSet<>(); Set<Integer> sectionsToReanalyse = new HashSet<>();
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
.stream() .stream()
.map(DictionaryIncrementValue::getValue) .map(DictionaryIncrementValue::getValue)
.toList(), true); .toList(), true);
List<String> unprocessedGroupRedactions = groupAnnotations.stream()
.filter(groupAnnotation -> groupAnnotation.getGroupAnnotationType().equals(GroupAnnotationType.TEXT))
.filter(UnprocessedUtils::isUnprocessedGroupAnnotation)
.map(GroupAnnotation::getValue)
.toList();
var groupRedactionSearch = new SearchImplementation(unprocessedGroupRedactions, false);
document.streamChildren() document.streamChildren()
.forEach(mainNode -> { .forEach(mainNode -> {
if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText()) || groupRedactionSearch.atLeastOneMatches(mainNode.getTextBlock() if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
.getSearchText())) {
sectionsToReanalyse.add(mainNode.getTreeId() sectionsToReanalyse.add(mainNode.getTreeId()
.get(0)); .get(0));
} }

38
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java
View File

@ -1,38 +0,0 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupChange;
import lombok.experimental.UtilityClass;
@UtilityClass
public class UnprocessedUtils {
public static boolean isUnprocessedGroupAnnotation(GroupAnnotation groupRedaction) {
if (groupRedaction.getGroupChanges().isEmpty() && groupRedaction.getProcessedDate() == null) {
return true;
}
if (!groupRedaction.getGroupChanges().isEmpty()) {
OffsetDateTime processedDate = groupRedaction.getProcessedDate();
if (processedDate == null) {
return true;
}
OffsetDateTime lastChangeDate = groupRedaction.getGroupChanges()
.stream()
.map(GroupChange::getChangeDate)
.max(OffsetDateTime::compareTo)
.orElse(null);
return lastChangeDate != null && lastChangeDate.isAfter(processedDate);
}
return false;
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -51,7 +51,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
@ -212,9 +211,6 @@ public abstract class AbstractRedactionIntegrationTest {
@MockBean @MockBean
protected DictionaryClient dictionaryClient; protected DictionaryClient dictionaryClient;
@MockBean
protected GroupRedactionClient groupRedactionClient;
@MockBean @MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
View File

@ -5,8 +5,6 @@ import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTe
import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME; import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME;
import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder; import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.File; import java.io.File;
@ -14,7 +12,6 @@ import java.io.FileInputStream;
import java.nio.file.FileVisitOption; import java.nio.file.FileVisitOption;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -51,9 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@ -106,8 +101,6 @@ import lombok.extern.slf4j.Slf4j;
@MockBean @MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@MockBean @MockBean
protected GroupRedactionClient groupRedactionClient;
@MockBean
private RedisSyncedWebSocketService redisSyncedWebSocketService; private RedisSyncedWebSocketService redisSyncedWebSocketService;
@MockBean @MockBean
private RedisMessageListenerContainer redisPubsubContainer; private RedisMessageListenerContainer redisPubsubContainer;
@ -188,10 +181,6 @@ import lombok.extern.slf4j.Slf4j;
String type = invocation.getArgument(0); String type = invocation.getArgument(0);
return testDossierTemplate.testDictionary.isHint(type); return testDossierTemplate.testDictionary.isHint(type);
}); });
when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder()
.textGroupAnnotations(new ArrayList<>())
.areaGroupAnnotations(new ArrayList<>())
.build());
when(dictionaryService.getColor(any(String.class), any())).thenAnswer(invocation -> { when(dictionaryService.getColor(any(String.class), any())).thenAnswer(invocation -> {
String type = invocation.getArgument(0); String type = invocation.getArgument(0);
return testDossierTemplate.testDictionary.getType(type).getColor(); return testDossierTemplate.testDictionary.getType(type).getColor();

28
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -3,8 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -41,7 +40,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@ -53,8 +52,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"}) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
@Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class) @Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class)
@ -232,11 +229,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
mockDictionaryCalls(null); mockDictionaryCalls(null);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder()
.textGroupAnnotations(Collections.emptyList())
.areaGroupAnnotations(Collections.emptyList())
.build());
} }
@ -282,7 +274,9 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
.fileId(TEST_FILE_ID) .fileId(TEST_FILE_ID)
.build()); .build());
}); });
request.setManualRedactions(ManualRedactions.builder().idsToRemove(idRemovalSet).build()); request.setManualRedactions(ManualRedactions.builder()
.idsToRemove(idRemovalSet)
.build());
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
@ -310,20 +304,12 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis"); System.out.println("Finished analysis");
var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var doseMortality = componentLog.getComponentLogEntries() var doseMortality = componentLog.getComponentLogEntries().stream().filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality")).findFirst().get();
.stream()
.filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality"))
.findFirst()
.get();
assertEquals(doseMortality.getComponentValues().size(), 5); assertEquals(doseMortality.getComponentValues().size(), 5);
Pattern pattern = Pattern.compile("^5000, [SD]$"); Pattern pattern = Pattern.compile("^5000, [SD]$");
boolean allMatch = doseMortality.getComponentValues() boolean allMatch = doseMortality.getComponentValues().stream().map(ComponentLogEntryValue::getValue).allMatch(pattern.asPredicate());
.stream()
.map(ComponentLogEntryValue::getValue)
.allMatch(pattern.asPredicate());
assertTrue(allMatch); assertTrue(allMatch);
} }
} }

158
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java
View File

@ -1,158 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(GroupRedactionTest.GroupRedactionTestConfiguration.class)
public class GroupRedactionTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class GroupRedactionTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
loadDictionaryForTest();
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse());
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse());
mockDictionaryCalls(null);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@Test
@SneakyThrows
public void testGroupRedaction() {
String textID = UUID.randomUUID().toString();
String areaID = UUID.randomUUID().toString();
TextGroupAnnotation textGroupRedaction = TextGroupAnnotation.builder()
.positionOnPage(PositionOnPage.builder().x(423.9f).y(505.5f).width(43.9f).height(15.4f).pageNumber(1).build())
.groupId(textID)
.typeId("CBI_author")
.value("medicine")
.legalBasis("test legal basis")
.entryType(EntryType.ENTITY)
.groupAnnotationType(GroupAnnotationType.TEXT)
.build();
AreaGroupAnnotation areaGroupRedaction = AreaGroupAnnotation.builder()
.positionOnPage(PositionOnPage.builder().x(346.27f).y(644.24f).width(219.4f).height(52.24f).pageNumber(2).build())
.groupId(areaID)
.pageRanges(List.of(PageRange.builder().startPage(2).endPage(5).build()))
.typeId("PII")
.value("non-readable content")
.entryType(EntryType.AREA)
.groupAnnotationType(GroupAnnotationType.AREA)
.section("my section")
.build();
when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder()
.textGroupAnnotations(List.of(textGroupRedaction))
.areaGroupAnnotations(List.of(areaGroupRedaction))
.build());
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
analyzeService.analyze(request);
System.out.println("Finished analysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
var textGroupRedactions = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getGroupId() != null && e.getGroupId().equals(textID))
.toList();
assertEquals(textGroupRedactions.size(), 6);
var areaGroupRedactions = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getGroupId() != null && e.getGroupId().equals(areaID))
.toList();
assertEquals(areaGroupRedactions.size(), 4);
}
}

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -47,7 +45,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@ -60,7 +58,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@ -109,11 +106,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
mockDictionaryCalls(null); mockDictionaryCalls(null);
when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder()
.textGroupAnnotations(Collections.emptyList())
.areaGroupAnnotations(Collections.emptyList())
.build());
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -68,7 +67,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
@ -85,7 +83,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@ -132,11 +129,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
mockDictionaryCalls(null); mockDictionaryCalls(null);
when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder()
.textGroupAnnotations(Collections.emptyList())
.areaGroupAnnotations(Collections.emptyList())
.build());
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -3,8 +3,7 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
@ -42,7 +41,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
@ -56,7 +55,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@ -107,11 +105,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse());
mockDictionaryCalls(null); mockDictionaryCalls(null);
when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder()
.textGroupAnnotations(Collections.emptyList())
.areaGroupAnnotations(Collections.emptyList())
.build());
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }