diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 18b95727..db05bba6 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.504.0" +val persistenceServiceVersion = "2.522.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 76e115bc..043426a5 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.141.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.509.0" +val persistenceServiceVersion = "2.522.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java new file mode 100644 index 00000000..fb6dd803 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import org.springframework.cloud.openfeign.FeignClient; + +import com.iqser.red.service.persistence.service.v1.api.internal.resources.GroupRedactionResource; + +@FeignClient(name = "GroupRedactionResource", url = "${persistence-service.url}") +public interface GroupRedactionClient extends GroupRedactionResource { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index a3ec0376..53a83b40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model; import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.PriorityQueue; @@ -14,11 +15,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -48,6 +51,7 @@ public class PrecursorEntity implements IEntity { boolean isDossierDictionaryEntry; boolean rectangle; Set engines; + String groupId; @Builder.Default PriorityQueue matchedRuleList = new PriorityQueue<>(); @@ -149,6 +153,49 @@ public class PrecursorEntity implements IEntity { } + public static List fromAreaGroupRedaction(AreaGroupRedaction areaGroupRedaction) { + + List precursorEntities = new ArrayList<>(); + + areaGroupRedaction.getPageRanges() + .forEach(pageRange -> { + for (int pageStart = pageRange.getStartPage(); pageStart <= pageRange.getEndPage(); pageStart++) { + boolean isHint = areaGroupRedaction.getEntryType() == EntryType.HINT; + String ruleIdentifier = isHint ? "GRP.5.0" : "GRP.5.1"; + String reason = isHint ? "group hint is skipped by default" : "group area is applied by default"; + String legalBasis = isHint ? "" : areaGroupRedaction.getLegalBasis(); + MatchedRule.MatchedRuleBuilder matchedRule = MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason); + if (!isHint) { + matchedRule.legalBasis(legalBasis); + matchedRule.applied(true); + } + precursorEntities.add(PrecursorEntity.builder() + .id(UUID.randomUUID().toString()) + .value(areaGroupRedaction.getValue()) + .entityPosition(List.of(RectangleWithPage.fromPositionOnPage(pageStart, areaGroupRedaction.getPositionOnPage()))) + .ruleIdentifier(ruleIdentifier) + .matchedRuleList(new PriorityQueue<>(List.of(matchedRule.build()))) + .reason(reason) + .legalBasis(legalBasis) + .section(areaGroupRedaction.getSection()) + .type(areaGroupRedaction.getTypeId()) + .entityType(isHint ? EntityType.HINT : EntityType.ENTITY) + .applied(!isHint) + .isDictionaryEntry(false) + .isDossierDictionaryEntry(false) + .rectangle(true) + .section(areaGroupRedaction.getSection()) + .engines(Set.of(Engine.GROUP)) + .groupId(areaGroupRedaction.getGroupId()) + .manualOverwrite(new ManualChangeOverwrite(isHint ? EntityType.HINT : EntityType.ENTITY)) + .build()); + } + }); + + return precursorEntities; + } + + public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { List rectangleWithPages = manualResizeRedaction.getPositions() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java index 447dd4d9..a85c4093 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.model; import java.awt.geom.Rectangle2D; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { @@ -19,6 +20,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { } + public static RectangleWithPage fromPositionOnPage(int pageNumber, PositionOnPage positionOnPage) { + + return new RectangleWithPage(pageNumber, toRectangle2D(positionOnPage)); + } + + public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) { return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle)); @@ -42,4 +49,10 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()); } + + private static Rectangle2D toRectangle2D(PositionOnPage positionOnPage) { + + return new Rectangle2D.Float(positionOnPage.getX(), positionOnPage.getY(), positionOnPage.getWidth(), positionOnPage.getHeight()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java index d68ab337..44d57070 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java @@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.Setter; import lombok.experimental.FieldDefaults; @Builder diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index a72a9adf..8fdd4179 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -29,7 +29,9 @@ public final class MatchedRule implements Comparable { public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP"); public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT"); - private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE); + public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN"); + public static final RuleType GROUP_TYPE = RuleType.fromString("GRP"); + private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE, MANUAL_TYPE, GROUP_TYPE); RuleIdentifier ruleIdentifier; @Builder.Default diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index 1c85d383..ff5abb9e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -49,6 +49,7 @@ public class TextEntity implements IEntity { boolean dictionaryEntry; boolean dossierDictionaryEntry; + String groupId; @Builder.Default Set engines = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java index 65a16c5c..40fded48 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java @@ -6,9 +6,11 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.task.TaskExecutor; @@ -18,7 +20,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.NerEntities; @@ -28,6 +36,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.document.AreaGroupRedactionService; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; @@ -36,6 +45,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils; import lombok.AccessLevel; import lombok.SneakyThrows; @@ -55,7 +65,9 @@ public class AnalysisPreparationService { ImportedRedactionEntryService importedRedactionEntryService; DictionaryService dictionaryService; SectionFinderService sectionFinderService; + GroupRedactionClient groupRedactionClient; TaskExecutor taskExecutor; + AreaGroupRedactionService areaGroupRedactionService; public AnalysisPreparationService(KieContainerCreationService kieContainerCreationService, @@ -66,7 +78,9 @@ public class AnalysisPreparationService { ImportedRedactionEntryService importedRedactionEntryService, DictionaryService dictionaryService, SectionFinderService sectionFinderService, - @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) { + GroupRedactionClient groupRedactionClient, + @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor, + AreaGroupRedactionService areaGroupRedactionService) { this.kieContainerCreationService = kieContainerCreationService; this.observedStorageService = observedStorageService; @@ -76,7 +90,9 @@ public class AnalysisPreparationService { this.importedRedactionEntryService = importedRedactionEntryService; this.dictionaryService = dictionaryService; this.sectionFinderService = sectionFinderService; + this.groupRedactionClient = groupRedactionClient; this.taskExecutor = taskExecutor; + this.areaGroupRedactionService = areaGroupRedactionService; } @@ -93,6 +109,8 @@ public class AnalysisPreparationService { CompletableFuture nerEntitiesFuture = documentFuture.thenApplyAsync((document) -> getNerEntities(analyzeRequest, document), taskExecutor); + CompletableFuture internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupRedactions(analyzeRequest, false), taskExecutor); + CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join(); Dictionary dictionary = getDictionary(analyzeRequest); @@ -106,6 +124,8 @@ public class AnalysisPreparationService { List notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); + List areaGroupRedactions = areaGroupRedactionService.createAreaGroupRedactions(internalResponseFuture.get().getAreaGroupRedactions()); + return new AnalysisData(kieWrapperEntityRulesFuture.get(), kieWrapperComponentRulesFuture.get(), document, @@ -113,7 +133,9 @@ public class AnalysisPreparationService { dictionary, notFoundManualRedactionEntries, notFoundImportedEntries, - nerEntitiesFuture.get()); + nerEntitiesFuture.get(), + internalResponseFuture.get(), + areaGroupRedactions); } @@ -135,11 +157,35 @@ public class AnalysisPreparationService { CompletableFuture importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor); - CompletableFuture incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenApplyAsync((importedRedactions) -> { - DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(analyzeRequest, reanalysisSetupData); - return getDictionaryIncrementAndSectionsToReanalyze(analyzeRequest, dictionaryIncrement, reanalysisSetupData, importedRedactions); + CompletableFuture internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupRedactions(analyzeRequest, true), taskExecutor); - }, taskExecutor); + CompletableFuture incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenCombineAsync(internalResponseFuture, + (importedRedactions, groupRedactions) -> { + List areaGroupRedactions = groupRedactions.getAreaGroupRedactions(); + List textGroupRedactions = groupRedactions.getTextGroupRedactions(); + + List internalGroupRedactions = Stream.concat( + Optional.ofNullable( + areaGroupRedactions) + .orElseGet(List::of) + .stream(), + Optional.ofNullable( + textGroupRedactions) + .orElseGet(List::of) + .stream()) + .toList(); + DictionaryIncrement dictionaryIncrement = getDictionaryIncrement( + analyzeRequest, + reanalysisSetupData); + return getDictionaryIncrementAndSectionsToReanalyze( + analyzeRequest, + dictionaryIncrement, + reanalysisSetupData, + importedRedactions, + internalGroupRedactions); + + }, + taskExecutor); CompletableFuture kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor); @@ -147,19 +193,26 @@ public class AnalysisPreparationService { CompletableFuture.allOf(importedRedactionsFuture, incrementAndSectionsToReanalyzeFuture, kieWrapperComponentRulesFuture, kieWrapperEntityRulesFuture).join(); + List areaGroupRedactions = internalResponseFuture.get().getAreaGroupRedactions().stream() + .filter(UnprocessedUtils::isUnprocessedGroupedRedaction) + .toList(); + return new ReanalysisInitialProcessingData(importedRedactionsFuture.get(), incrementAndSectionsToReanalyzeFuture.get().dictionaryIncrement(), incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyseIds(), incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyze(), kieWrapperComponentRulesFuture.get(), - kieWrapperEntityRulesFuture.get()); + kieWrapperEntityRulesFuture.get(), + internalResponseFuture.get(), + areaGroupRedactions); } @SneakyThrows public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData, - ReanalysisInitialProcessingData reanalysisInitialProcessingData) { + ReanalysisInitialProcessingData reanalysisInitialProcessingData, + GroupRedactionInternalResponse groupRedactions) { CompletableFuture nerEntitiesFuture = CompletableFuture.supplyAsync(() -> getNerEntitiesFiltered(analyzeRequest, reanalysisSetupData.document, @@ -168,16 +221,25 @@ public class AnalysisPreparationService { CompletableFuture dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> { Dictionary dictionary = getDictionary(analyzeRequest); - NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions()); - return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries()); + NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, + reanalysisSetupData.document(), + reanalysisInitialProcessingData.importedRedactions(), + groupRedactions.getAreaGroupRedactions().stream().filter(gR -> gR.getSoftDeletedTime() == null).toList()); + return new DictionaryAndNotFoundEntries(dictionary, + notFoundEntries.notFoundManualRedactionEntries(), + notFoundEntries.notFoundImportedEntries(), + notFoundEntries.areaGroupRedactions()); }, taskExecutor); - CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture).join(); + CompletableFuture internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupRedactions(analyzeRequest, false), taskExecutor); + + CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture, internalResponseFuture).join(); return new ReanalysisFinalProcessingData(nerEntitiesFuture.get(), dictionaryAndNotFoundEntriesCompletableFuture.get().dictionary(), dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundManualRedactionEntries(), - dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries()); + dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries(), + dictionaryAndNotFoundEntriesCompletableFuture.get().areaGroupRedactions()); } @@ -201,6 +263,14 @@ public class AnalysisPreparationService { } + public GroupRedactionInternalResponse getGroupRedactions(AnalyzeRequest analyzeRequest, boolean includeDeleted) { + + GroupRedactionInternalResponse groupRedactionResponse = groupRedactionClient.getGroupRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), includeDeleted); + log.info("Loaded group redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + return groupRedactionResponse; + } + + public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) { ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); @@ -234,13 +304,14 @@ public class AnalysisPreparationService { } - private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) { + private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions, List groupRedactions) { var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document, analyzeRequest.getDossierTemplateId()); var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); - return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries); + var areaGroupRedactions = areaGroupRedactionService.createAreaGroupRedactions(groupRedactions); + return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries, areaGroupRedactions); } @@ -267,7 +338,8 @@ public class AnalysisPreparationService { private SectionsToReanalyzeData getDictionaryIncrementAndSectionsToReanalyze(AnalyzeRequest analyzeRequest, DictionaryIncrement dictionaryIncrement, ReanalysisSetupData reanalysisSetupData, - ImportedRedactions importedRedactions) { + ImportedRedactions importedRedactions, + List groupRedactions) { Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); @@ -278,7 +350,8 @@ public class AnalysisPreparationService { reanalysisSetupData.document(), dictionaryIncrement, importedRedactions, - relevantManuallyModifiedAnnotationIds)); + relevantManuallyModifiedAnnotationIds, + groupRedactions)); List sectionsToReAnalyse = getSectionsToReAnalyse(reanalysisSetupData.document(), sectionsToReanalyseIds); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -339,21 +412,33 @@ public class AnalysisPreparationService { Document document, DictionaryIncrement dictionaryIncrement, ImportedRedactions importedRedactions, - Set relevantManuallyModifiedAnnotationIds) { + Set relevantManuallyModifiedAnnotationIds, + List groupRedactions) { - return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds); + return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, + document, + analyzeRequest, + importedRedactions, + relevantManuallyModifiedAnnotationIds, + groupRedactions); } - private record DictionaryAndNotFoundEntries(Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries) { + private record DictionaryAndNotFoundEntries( + Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries, List areaGroupRedactions + ) { } - private record NotFoundEntries(List notFoundManualRedactionEntries, List notFoundImportedEntries) { + private record NotFoundEntries(List notFoundManualRedactionEntries, List notFoundImportedEntries, List areaGroupRedactions) { } - private record SectionsToReanalyzeData(DictionaryIncrement dictionaryIncrement, Set sectionsToReanalyseIds, List sectionsToReanalyze) { + private record SectionsToReanalyzeData( + DictionaryIncrement dictionaryIncrement, + Set sectionsToReanalyseIds, + List sectionsToReanalyze + ) { } @@ -365,7 +450,9 @@ public class AnalysisPreparationService { Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries, - NerEntities nerEntities + NerEntities nerEntities, + GroupRedactionInternalResponse groupRedactions, + List areaGroupRedactions ) { } @@ -382,13 +469,19 @@ public class AnalysisPreparationService { Set sectionsToReanalyseIds, List sectionsToReAnalyse, KieWrapper kieWrapperComponentRules, - KieWrapper kieWrapperEntityRules + KieWrapper kieWrapperEntityRules, + GroupRedactionInternalResponse groupRedactions, + List unprocessedAreaGroupRedactions ) { } public record ReanalysisFinalProcessingData( - NerEntities nerEntities, Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries + NerEntities nerEntities, + Dictionary dictionary, + List notFoundManualRedactionEntries, + List notFoundImportedEntries, + List areaGroupRedactions ) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index d5a4ba19..643bc4ab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -9,6 +9,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,8 +25,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.logger.Context; @@ -64,6 +68,7 @@ public class AnalyzeService { FunctionTimerValues redactmanagerAnalyzePagewiseValues; ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE; AnalysisPreparationService analysisPreparationService; + GroupSearchService groupSearchService; @Timed("redactmanager_reanalyze") @@ -88,7 +93,7 @@ public class AnalyzeService { ReanalysisInitialProcessingData initialProcessingData = analysisPreparationService.getReanalysisInitialProcessingData(analyzeRequest, setupData); - if (initialProcessingData.sectionsToReAnalyse().isEmpty()) { + if (initialProcessingData.sectionsToReAnalyse().isEmpty() && initialProcessingData.groupRedactions().getAreaGroupRedactions().isEmpty()) { EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(setupData.entityLog(), initialProcessingData.dictionaryIncrement().getDictionaryVersion(), @@ -96,6 +101,8 @@ public class AnalyzeService { new ArrayList<>(), new ArrayList<>()); + Set groupRedactions = getGroupRedactionsList(initialProcessingData.groupRedactions()); + return finalizeAnalysis(analyzeRequest, startTime, initialProcessingData.kieWrapperComponentRules(), @@ -104,17 +111,28 @@ public class AnalyzeService { setupData.document().getNumberOfPages(), true, new HashSet<>(), + groupRedactions, context); } context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion()); - ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, setupData, initialProcessingData); + ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, + setupData, + initialProcessingData, + initialProcessingData.groupRedactions()); - dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse()); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + if (!initialProcessingData.sectionsToReAnalyse().isEmpty()) { + dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse()); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - var notFoundManualOrImportedEntries = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), finalProcessingData.notFoundImportedEntries()) + groupSearchService.addGroupEntities(initialProcessingData.groupRedactions().getTextGroupRedactions(), initialProcessingData.sectionsToReAnalyse()); + log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + } + + var notFoundManualOrImportedEntriesOrAreaGroupRedactions = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), + finalProcessingData.notFoundImportedEntries(), + finalProcessingData.areaGroupRedactions()) .flatMap(Collection::stream) .collect(Collectors.toList()); @@ -132,12 +150,14 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, setupData.document(), setupData.entityLog(), - notFoundManualOrImportedEntries, + notFoundManualOrImportedEntriesOrAreaGroupRedactions, initialProcessingData.sectionsToReanalyseIds(), finalProcessingData.dictionary().getVersion()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries()); + Set groupRedactions = getGroupRedactionsList(initialProcessingData.groupRedactions()); + return finalizeAnalysis(analyzeRequest, startTime, initialProcessingData.kieWrapperComponentRules(), @@ -146,6 +166,7 @@ public class AnalyzeService { setupData.document().getNumberOfPages(), true, new HashSet<>(allFileAttributes), + groupRedactions, context); } @@ -166,13 +187,18 @@ public class AnalyzeService { analyzeRequest.getAnalysisNumber(), TenantContext.getTenantId()); - var notFoundManualOrImportedEntries = Stream.of(analysisData.notFoundManualRedactionEntries(), analysisData.notFoundImportedEntries()) + var notFoundManualOrImportedEntriesOrGroupAreaRedaction = Stream.of(analysisData.notFoundManualRedactionEntries(), + analysisData.notFoundImportedEntries(), + analysisData.areaGroupRedactions()) .flatMap(Collection::stream) .collect(Collectors.toList()); dictionarySearchService.addDictionaryEntities(analysisData.dictionary(), analysisData.document()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + groupSearchService.addGroupEntities(analysisData.groupRedactions().getTextGroupRedactions(), analysisData.document()); + log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + // we could add the imported redactions similar to the manual redactions here as well for additional processing List allFileAttributes = entityDroolsExecutionService.executeRules(analysisData.kieWrapperEntityRules().container(), analysisData.document(), @@ -185,12 +211,14 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, analysisData.document(), - notFoundManualOrImportedEntries, + notFoundManualOrImportedEntriesOrGroupAreaRedaction, analysisData.dictionary().getVersion(), analysisData.kieWrapperEntityRules().rulesVersion()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); + Set groupRedactions = getGroupRedactionsList(analysisData.groupRedactions()); + return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), @@ -199,9 +227,11 @@ public class AnalyzeService { analysisData.document().getNumberOfPages(), false, new HashSet<>(allFileAttributes), + groupRedactions, context); } + @Timed("redactmanager_analyzeImportedRedactionsOnly") @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly") public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) { @@ -237,7 +267,17 @@ public class AnalyzeService { notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); - return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), entityLogChanges, analysisData.document(), analysisData.document().getNumberOfPages(), false, new HashSet<>(), + Set groupRedactions = getGroupRedactionsList(analysisData.groupRedactions()); + + return finalizeAnalysis(analyzeRequest, + startTime, + analysisData.kieWrapperComponentRules(), + entityLogChanges, + analysisData.document(), + analysisData.document().getNumberOfPages(), + false, + new HashSet<>(), + groupRedactions, context); } @@ -250,6 +290,7 @@ public class AnalyzeService { int numberOfPages, boolean isReanalysis, Set addedFileAttributes, + Set groupRedactions, Context context) { EntityLog entityLog = entityLogChanges.getEntityLog(); @@ -295,6 +336,7 @@ public class AnalyzeService { .manualRedactions(analyzeRequest.getManualRedactions()) .addedFileAttributes(addedFileAttributes) .usedComponentMappings(analyzeRequest.getComponentMappings()) + .groupRedactions(groupRedactions) .build(); } @@ -329,4 +371,19 @@ public class AnalyzeService { log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } + + private Set getGroupRedactionsList(GroupRedactionInternalResponse groupRedactionInternalResponse) { + + List areaGroupRedactions = groupRedactionInternalResponse.getAreaGroupRedactions(); + List textGroupRedactions = groupRedactionInternalResponse.getTextGroupRedactions(); + + return Stream.concat(Optional.ofNullable(areaGroupRedactions) + .orElseGet(List::of) + .stream(), + Optional.ofNullable(textGroupRedactions) + .orElseGet(List::of) + .stream()) + .collect(Collectors.toSet()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index e1be0b90..fb8800f7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange; @@ -44,10 +45,25 @@ public class EntityChangeLogService { List toInsert = new ArrayList<>(); List toUpdate = new ArrayList<>(); + List existingAreaGroupEntries = new ArrayList<>(); + for (EntityLogEntry entityLogEntry : newEntityLogEntries) { Optional optionalPreviousEntity = previousEntityLogEntries.stream() .filter(entry -> entry.getId().equals(entityLogEntry.getId())) .findAny(); + + Optional optionalPreviousAreaGroupEntity = previousEntityLogEntries.stream() + .filter(entry -> entry.getGroupId() != null && entry.getGroupId().equals(entityLogEntry.getGroupId())) + .filter(entry -> entry.getEntryType().equals(EntryType.AREA)) + .filter(entry -> entry.getPositions().equals(entityLogEntry.getPositions())) + .findAny(); + + // Skip group area entities on the same position + if (optionalPreviousAreaGroupEntity.isPresent()) { + existingAreaGroupEntries.add(optionalPreviousAreaGroupEntity.get()); + continue; + } + if (optionalPreviousEntity.isEmpty()) { entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())); toInsert.add(entityLogEntry); @@ -65,19 +81,21 @@ public class EntityChangeLogService { } } - toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now)); + toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, existingAreaGroupEntries, analysisNumber, now)); return new EntryChanges(toInsert, toUpdate); } private List addRemovedEntriesAsRemoved(List previousEntityLogEntries, - List newEntityLogEntries, - int analysisNumber, + List newEntityLogEntries, List existingAreaGroupEntries, int analysisNumber, OffsetDateTime now) { Set existingIds = newEntityLogEntries.stream() .map(EntityLogEntry::getId) .collect(Collectors.toSet()); + existingIds.addAll(existingAreaGroupEntries.stream() + .map(EntityLogEntry::getId) + .collect(Collectors.toSet())); List removedEntries = previousEntityLogEntries.stream() .filter(entry -> !existingIds.contains(entry.getId())) .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 182b93f5..55fa9776 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; @@ -238,7 +239,7 @@ public class EntityLogCreatorService { List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); List allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); - return EntityLogEntry.builder() + EntityLogEntry entityLogEntry = EntityLogEntry.builder() .id(precursorEntity.getId()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(precursorEntity.legalBasis()) @@ -267,9 +268,12 @@ public class EntityLogCreatorService { //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) .imported(false) .reference(Collections.emptySet()) - .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) + .manualChanges(precursorEntity.getGroupId() == null ? ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges) : Collections.emptyList()) + .groupId(precursorEntity.getGroupId()) .paragraphPageIdx(-1) .build(); + + return entityLogEntry; } @@ -318,6 +322,7 @@ public class EntityLogCreatorService { .state(buildEntryState(entity)) .entryType(entryType) .paragraphPageIdx(determinePageParagraphIndex(entity, entryType)) + .groupId(entity.getGroupId()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java new file mode 100644 index 00000000..0d8d8fb8 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java @@ -0,0 +1,92 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.List; +import java.util.Set; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class GroupSearchService { + + EntityEnrichmentService entityEnrichmentService; + + + public void addGroupEntities(List groupRedactions, List nodes) { + + nodes.forEach(node -> addGroupEntities(groupRedactions, node)); + } + + + public void addGroupEntities(List groupRedactions, SemanticNode node) { + + groupRedactions.stream() + .filter(groupRedaction -> groupRedaction.getSoftDeletedTime() == null) + .forEach(groupRedaction -> { + SearchImplementation searchImplementation = new SearchImplementation(groupRedaction.getValue(), false); + bySearchImplementationAsGroup(searchImplementation, + groupRedaction.getTypeId(), + calculateEntityType(groupRedaction.getEntryType()), + groupRedaction.getGroupId(), + groupRedaction.getLegalBasis(), + node); + }); + } + + + public void bySearchImplementationAsGroup(SearchImplementation searchImplementation, String type, EntityType entityType, String groupId, String legalBasis, SemanticNode node) { + + Set engines = Set.of(Engine.GROUP); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) + .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) + .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines) + .ifPresent(entity -> { + if (entity.getEntityType().equals(EntityType.HINT)) { + entity.skip("GRP.0.1", "group hint is skipped by default"); + } else { + entity.apply("GRP.0.0", "group entries are applied by default", legalBasis); + } + entity.setGroupId(groupId); + })); + } + + + private EntityType calculateEntityType(EntryType entryType) { + + switch (entryType) { + case FALSE_RECOMMENDATION -> { + return EntityType.FALSE_RECOMMENDATION; + } + case RECOMMENDATION -> { + return EntityType.RECOMMENDATION; + } + case HINT -> { + return EntityType.HINT; + } + case FALSE_POSITIVE -> { + return EntityType.FALSE_POSITIVE; + } + default -> { + return EntityType.ENTITY; + } + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupRedactionService.java new file mode 100644 index 00000000..2fd9a10e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupRedactionService.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class AreaGroupRedactionService { + + public List createAreaGroupRedactions(List groupRedactions) { + + List precursorEntities = new ArrayList<>(); + if (groupRedactions != null) { + groupRedactions.forEach(groupRedaction -> precursorEntities.addAll(PrecursorEntity.fromAreaGroupRedaction(groupRedaction))); + } + return precursorEntities; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index f45a3299..6bd498bc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -11,7 +11,6 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; @@ -23,11 +22,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionType; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -46,20 +48,30 @@ public class SectionFinderService { Document document, AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, - Set relevantManuallyModifiedAnnotationIds) { + Set relevantManuallyModifiedAnnotationIds, + List groupRedactions) { long start = System.currentTimeMillis(); Set sectionsToReanalyse = new HashSet<>(); - var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() .stream() .map(DictionaryIncrementValue::getValue) .toList(), true); + List unprocessedGroupRedactions = groupRedactions + .stream() + .filter(groupRedaction -> groupRedaction.getGroupRedactionType().equals(GroupRedactionType.TEXT)) + .filter(UnprocessedUtils::isUnprocessedGroupedRedaction) + .map(GroupRedaction::getValue) + .toList(); + + var groupRedactionSearch = new SearchImplementation(unprocessedGroupRedactions, false); + document.streamChildren() .forEach(mainNode -> { - if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) { + if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText()) || groupRedactionSearch.atLeastOneMatches(mainNode.getTextBlock() + .getSearchText())) { sectionsToReanalyse.add(mainNode.getTreeId() .get(0)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java new file mode 100644 index 00000000..7f559796 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.time.OffsetDateTime; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class UnprocessedUtils { + + + public static boolean isUnprocessedGroupedRedaction(GroupRedaction groupRedaction) { + + if (groupRedaction.getGroupChanges().isEmpty() && groupRedaction.getProcessedDate() == null) { + return true; + } + + if (!groupRedaction.getGroupChanges().isEmpty()) { + OffsetDateTime processedDate = groupRedaction.getProcessedDate(); + + if (processedDate == null) { + return true; + } + + OffsetDateTime lastChangeDate = groupRedaction.getGroupChanges().stream() + .map(GroupChange::getChangeDate) + .max(OffsetDateTime::compareTo) + .orElse(null); + + return lastChangeDate != null && lastChangeDate.isAfter(processedDate); + } + + return false; + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index fb412bd8..630d85f7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -48,6 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository. import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; @@ -201,6 +202,9 @@ public abstract class AbstractRedactionIntegrationTest { @MockBean protected DictionaryClient dictionaryClient; + @MockBean + protected GroupRedactionClient groupRedactionClient; + @MockBean protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index d96438fa..8dc388a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -41,8 +43,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; +import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; @@ -52,6 +56,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; +import edu.emory.mathcs.backport.java.util.Collections; + @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"}) @Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class) @@ -229,6 +235,11 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + + when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder() + .textGroupRedactions(Collections.emptyList()) + .areaGroupRedactions(Collections.emptyList()) + .build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java new file mode 100644 index 00000000..5a5051c4 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java @@ -0,0 +1,158 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +import java.io.FileOutputStream; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; +import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.SneakyThrows; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(GroupRedactionTest.GroupRedactionTestConfiguration.class) +public class GroupRedactionTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/rules.drl"); + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import({LayoutParsingServiceProcessorConfiguration.class}) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class GroupRedactionTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); + + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + @SneakyThrows + public void testGroupRedaction() { + + String textID = UUID.randomUUID().toString(); + String areaID = UUID.randomUUID().toString(); + + TextGroupRedaction textGroupRedaction = TextGroupRedaction.builder() + .positionOnPage(PositionOnPage.builder().x(423.9f).y(505.5f).width(43.9f).height(15.4f).pageNumber(1).build()) + .groupId(textID) + .typeId("CBI_author") + .value("medicine") + .legalBasis("test legal basis") + .entryType(EntryType.ENTITY) + .groupRedactionType(GroupRedactionType.TEXT) + .build(); + + AreaGroupRedaction areaGroupRedaction = AreaGroupRedaction.builder() + .positionOnPage(PositionOnPage.builder().x(346.27f).y(644.24f).width(219.4f).height(52.24f).pageNumber(2).build()) + .groupId(areaID) + .pageRanges(List.of(PageRange.builder().startPage(2).endPage(5).build())) + .typeId("PII") + .value("non-readable content") + .entryType(EntryType.AREA) + .groupRedactionType(GroupRedactionType.AREA) + .section("my section") + .build(); + + when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder() + .textGroupRedactions(List.of(textGroupRedaction)) + .areaGroupRedactions(List.of(areaGroupRedaction)) + .build()); + + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf"); + System.out.println("Start Full integration test"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + System.out.println("Finished structure analysis"); + analyzeService.analyze(request); + System.out.println("Finished analysis"); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + + String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; + + try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + var textGroupRedactions = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getGroupId() != null && e.getGroupId().equals(textID)) + .toList(); + assertEquals(textGroupRedactions.size(), 6); + + var areaGroupRedactions = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getGroupId() != null && e.getGroupId().equals(areaID)) + .toList(); + assertEquals(areaGroupRedactions.size(), 4); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index c7dad2bc..ebde3723 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -46,6 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; @@ -58,6 +61,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; +import edu.emory.mathcs.backport.java.util.Collections; import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @@ -106,6 +110,11 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { mockDictionaryCalls(null); + when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder() + .textGroupRedactions(Collections.emptyList()) + .areaGroupRedactions(Collections.emptyList()) + .build()); + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index ac368c12..6374f601 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; @@ -67,6 +68,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; @@ -83,6 +85,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; +import edu.emory.mathcs.backport.java.util.Collections; import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @@ -129,6 +132,11 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { mockDictionaryCalls(null); + when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder() + .textGroupRedactions(Collections.emptyList()) + .areaGroupRedactions(Collections.emptyList()) + .build()); + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index 6295723a..88faffb5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -42,6 +44,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse; import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; @@ -55,6 +58,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; +import edu.emory.mathcs.backport.java.util.Collections; import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @@ -105,6 +109,11 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); + when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder() + .textGroupRedactions(Collections.emptyList()) + .areaGroupRedactions(Collections.emptyList()) + .build()); + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); }