From b9240dec68518ee150b41ddc1a7f8d7a4f5beaa8 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Thu, 29 Aug 2024 10:18:48 +0200 Subject: [PATCH] Revert "RED-7327 - Change to group annotation" --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../build.gradle.kts | 2 +- .../server/client/GroupRedactionClient.java | 10 -- .../v1/server/model/PrecursorEntity.java | 47 ------ .../v1/server/model/RectangleWithPage.java | 13 -- .../entity/ManualChangeOverwrite.java | 1 - .../model/document/entity/MatchedRule.java | 4 +- .../model/document/entity/TextEntity.java | 1 - .../service/AnalysisPreparationService.java | 152 +++-------------- .../v1/server/service/AnalyzeService.java | 78 ++------- .../service/EntityChangeLogService.java | 24 +-- .../service/EntityLogCreatorService.java | 8 +- .../v1/server/service/GroupSearchService.java | 92 ---------- .../document/AreaGroupAnnotationService.java | 28 ---- .../document/SectionFinderService.java | 19 +-- .../v1/server/utils/UnprocessedUtils.java | 38 ----- .../AbstractRedactionIntegrationTest.java | 4 - .../v1/server/AnalysisEnd2EndTest.java | 11 -- .../v1/server/DocumineFloraTest.java | 28 +--- .../v1/server/GroupRedactionTest.java | 158 ------------------ .../v1/server/RedactionAcceptanceTest.java | 10 +- .../v1/server/RedactionIntegrationTest.java | 8 - .../UnprocessedChangesServiceTest.java | 11 +- 23 files changed, 57 insertions(+), 692 deletions(-) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupAnnotationService.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 78bc2b3f..94f0e54f 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.532.0" +val persistenceServiceVersion = "2.536.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index c625deb8..222b7995 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.161.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.532.0" +val persistenceServiceVersion = "2.536.0" val llmServiceVersion = "1.11.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java deleted file mode 100644 index c6f13baf..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.client; - -import org.springframework.cloud.openfeign.FeignClient; - -import com.iqser.red.service.persistence.service.v1.api.internal.resources.GroupAnnotationResource; - -@FeignClient(name = "GroupAnnotationResource", url = "${persistence-service.url}") -public interface GroupRedactionClient extends GroupAnnotationResource { - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index 566785ef..a3ec0376 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model; import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.PriorityQueue; @@ -15,13 +14,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -51,7 +48,6 @@ public class PrecursorEntity implements IEntity { boolean isDossierDictionaryEntry; boolean rectangle; Set engines; - String groupId; @Builder.Default PriorityQueue matchedRuleList = new PriorityQueue<>(); @@ -153,49 +149,6 @@ public class PrecursorEntity implements IEntity { } - public static List fromAreaGroupAnnotation(AreaGroupAnnotation areaGroupAnnotation) { - - List precursorEntities = new ArrayList<>(); - - areaGroupAnnotation.getPageRanges() - .forEach(pageRange -> { - for (int pageStart = pageRange.getStartPage(); pageStart <= pageRange.getEndPage(); pageStart++) { - boolean isHint = areaGroupAnnotation.getEntryType() == EntryType.HINT; - String ruleIdentifier = isHint ? "GRP.5.0" : "GRP.5.1"; - String reason = isHint ? "group hint is skipped by default" : "group area is applied by default"; - String legalBasis = isHint ? "" : areaGroupAnnotation.getLegalBasis(); - MatchedRule.MatchedRuleBuilder matchedRule = MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason); - if (!isHint) { - matchedRule.legalBasis(legalBasis); - matchedRule.applied(true); - } - precursorEntities.add(PrecursorEntity.builder() - .id(UUID.randomUUID().toString()) - .value(areaGroupAnnotation.getValue()) - .entityPosition(List.of(RectangleWithPage.fromPositionOnPage(pageStart, areaGroupAnnotation.getPositionOnPage()))) - .ruleIdentifier(ruleIdentifier) - .matchedRuleList(new PriorityQueue<>(List.of(matchedRule.build()))) - .reason(reason) - .legalBasis(legalBasis) - .section(areaGroupAnnotation.getSection()) - .type(areaGroupAnnotation.getTypeId()) - .entityType(isHint ? EntityType.HINT : EntityType.ENTITY) - .applied(!isHint) - .isDictionaryEntry(false) - .isDossierDictionaryEntry(false) - .rectangle(true) - .section(areaGroupAnnotation.getSection()) - .engines(Set.of(Engine.GROUP)) - .groupId(areaGroupAnnotation.getGroupId()) - .manualOverwrite(new ManualChangeOverwrite(isHint ? EntityType.HINT : EntityType.ENTITY)) - .build()); - } - }); - - return precursorEntities; - } - - public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { List rectangleWithPages = manualResizeRedaction.getPositions() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java index a85c4093..447dd4d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.model; import java.awt.geom.Rectangle2D; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { @@ -20,12 +19,6 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { } - public static RectangleWithPage fromPositionOnPage(int pageNumber, PositionOnPage positionOnPage) { - - return new RectangleWithPage(pageNumber, toRectangle2D(positionOnPage)); - } - - public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) { return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle)); @@ -49,10 +42,4 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) { return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()); } - - private static Rectangle2D toRectangle2D(PositionOnPage positionOnPage) { - - return new Rectangle2D.Float(positionOnPage.getX(), positionOnPage.getY(), positionOnPage.getWidth(), positionOnPage.getHeight()); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java index 44d57070..d68ab337 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java @@ -19,7 +19,6 @@ import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Setter; import lombok.experimental.FieldDefaults; @Builder diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index 8fdd4179..a72a9adf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -29,9 +29,7 @@ public final class MatchedRule implements Comparable { public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP"); public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT"); - public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN"); - public static final RuleType GROUP_TYPE = RuleType.fromString("GRP"); - private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE, MANUAL_TYPE, GROUP_TYPE); + private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE); RuleIdentifier ruleIdentifier; @Builder.Default diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index 1f5418c8..a80a00c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -53,7 +53,6 @@ public class TextEntity implements IEntity { boolean dictionaryEntry; boolean dossierDictionaryEntry; - String groupId; @Builder.Default Set engines = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java index 68cd01f8..1bdfe1b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java @@ -6,11 +6,9 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.task.TaskExecutor; @@ -20,12 +18,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; -import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.NerEntities; @@ -36,7 +29,6 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.service.document.AreaGroupAnnotationService; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; @@ -45,7 +37,6 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils; import lombok.AccessLevel; import lombok.SneakyThrows; @@ -65,9 +56,7 @@ public class AnalysisPreparationService { ImportedRedactionEntryService importedRedactionEntryService; DictionaryService dictionaryService; SectionFinderService sectionFinderService; - GroupRedactionClient groupRedactionClient; TaskExecutor taskExecutor; - AreaGroupAnnotationService areaGroupAnnotationService; public AnalysisPreparationService(KieContainerCreationService kieContainerCreationService, @@ -78,9 +67,7 @@ public class AnalysisPreparationService { ImportedRedactionEntryService importedRedactionEntryService, DictionaryService dictionaryService, SectionFinderService sectionFinderService, - GroupRedactionClient groupRedactionClient, - @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor, - AreaGroupAnnotationService areaGroupAnnotationService) { + @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) { this.kieContainerCreationService = kieContainerCreationService; this.observedStorageService = observedStorageService; @@ -90,9 +77,7 @@ public class AnalysisPreparationService { this.importedRedactionEntryService = importedRedactionEntryService; this.dictionaryService = dictionaryService; this.sectionFinderService = sectionFinderService; - this.groupRedactionClient = groupRedactionClient; this.taskExecutor = taskExecutor; - this.areaGroupAnnotationService = areaGroupAnnotationService; } @@ -109,10 +94,6 @@ public class AnalysisPreparationService { CompletableFuture nerEntitiesFuture = documentFuture.thenApplyAsync((document) -> getNerEntities(analyzeRequest, document), taskExecutor); - CompletableFuture groupAnnotationInternalResponseCompletableFuture = CompletableFuture.supplyAsync(() -> getGroupAnnotations(analyzeRequest, - false), - taskExecutor); - CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join(); Dictionary dictionary = getDictionary(analyzeRequest); @@ -126,9 +107,6 @@ public class AnalysisPreparationService { List notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); - List areaGroupRedactions = areaGroupAnnotationService.createAreaGroupAnnotations(groupAnnotationInternalResponseCompletableFuture.get() - .getAreaGroupAnnotations()); - return new AnalysisData(kieWrapperEntityRulesFuture.get(), kieWrapperComponentRulesFuture.get(), document, @@ -136,9 +114,7 @@ public class AnalysisPreparationService { dictionary, notFoundManualRedactionEntries, notFoundImportedEntries, - nerEntitiesFuture.get(), - groupAnnotationInternalResponseCompletableFuture.get(), - areaGroupRedactions); + nerEntitiesFuture.get()); } @@ -160,35 +136,11 @@ public class AnalysisPreparationService { CompletableFuture importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor); - CompletableFuture internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupAnnotations(analyzeRequest, true), taskExecutor); + CompletableFuture incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenApplyAsync((importedRedactions) -> { + DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(analyzeRequest, reanalysisSetupData); + return getDictionaryIncrementAndSectionsToReanalyze(analyzeRequest, dictionaryIncrement, reanalysisSetupData, importedRedactions); - CompletableFuture incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenCombineAsync(internalResponseFuture, - (importedRedactions, groupRedactions) -> { - List areaGroupRedactions = groupRedactions.getAreaGroupAnnotations(); - List textGroupRedactions = groupRedactions.getTextGroupAnnotations(); - - List groupAnnotations = Stream.concat( - Optional.ofNullable( - areaGroupRedactions) - .orElseGet(List::of) - .stream(), - Optional.ofNullable( - textGroupRedactions) - .orElseGet(List::of) - .stream()) - .toList(); - DictionaryIncrement dictionaryIncrement = getDictionaryIncrement( - analyzeRequest, - reanalysisSetupData); - return getDictionaryIncrementAndSectionsToReanalyze( - analyzeRequest, - dictionaryIncrement, - reanalysisSetupData, - importedRedactions, - groupAnnotations); - - }, - taskExecutor); + }, taskExecutor); CompletableFuture kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor); @@ -196,27 +148,19 @@ public class AnalysisPreparationService { CompletableFuture.allOf(importedRedactionsFuture, incrementAndSectionsToReanalyzeFuture, kieWrapperComponentRulesFuture, kieWrapperEntityRulesFuture).join(); - List areaGroupAnnotations = internalResponseFuture.get().getAreaGroupAnnotations() - .stream() - .filter(UnprocessedUtils::isUnprocessedGroupAnnotation) - .toList(); - return new ReanalysisInitialProcessingData(importedRedactionsFuture.get(), incrementAndSectionsToReanalyzeFuture.get().dictionaryIncrement(), incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyseIds(), incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyze(), kieWrapperComponentRulesFuture.get(), - kieWrapperEntityRulesFuture.get(), - internalResponseFuture.get(), - areaGroupAnnotations); + kieWrapperEntityRulesFuture.get()); } @SneakyThrows public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData, - ReanalysisInitialProcessingData reanalysisInitialProcessingData, - GroupAnnotationInternalResponse groupAnnotationInternalResponse) { + ReanalysisInitialProcessingData reanalysisInitialProcessingData) { CompletableFuture nerEntitiesFuture = CompletableFuture.supplyAsync(() -> getNerEntitiesFiltered(analyzeRequest, reanalysisSetupData.document, @@ -225,30 +169,16 @@ public class AnalysisPreparationService { CompletableFuture dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> { Dictionary dictionary = getDictionary(analyzeRequest); - NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, - reanalysisSetupData.document(), - reanalysisInitialProcessingData.importedRedactions(), - groupAnnotationInternalResponse.getAreaGroupAnnotations() - .stream() - .filter(gR -> gR.getSoftDeletedTime() == null) - .toList()); - return new DictionaryAndNotFoundEntries(dictionary, - notFoundEntries.notFoundManualRedactionEntries(), - notFoundEntries.notFoundImportedEntries(), - notFoundEntries.areaGroupRedactions()); + NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions()); + return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries()); }, taskExecutor); - CompletableFuture groupAnnotationInternalResponseCompletableFuture = CompletableFuture.supplyAsync(() -> getGroupAnnotations(analyzeRequest, - false), - taskExecutor); - - CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture, groupAnnotationInternalResponseCompletableFuture).join(); + CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture).join(); return new ReanalysisFinalProcessingData(nerEntitiesFuture.get(), dictionaryAndNotFoundEntriesCompletableFuture.get().dictionary(), dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundManualRedactionEntries(), - dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries(), - dictionaryAndNotFoundEntriesCompletableFuture.get().areaGroupRedactions()); + dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries()); } @@ -272,16 +202,6 @@ public class AnalysisPreparationService { } - public GroupAnnotationInternalResponse getGroupAnnotations(AnalyzeRequest analyzeRequest, boolean includeDeleted) { - - GroupAnnotationInternalResponse groupRedactionResponse = groupRedactionClient.getGroupAnnotations(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - includeDeleted); - log.info("Loaded group redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - return groupRedactionResponse; - } - - public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) { ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); @@ -315,17 +235,13 @@ public class AnalysisPreparationService { } - private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, - Document document, - ImportedRedactions importedRedactions, - List groupAnnotations) { + private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) { var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document, analyzeRequest.getDossierTemplateId()); var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); - var areaGroupRedactions = areaGroupAnnotationService.createAreaGroupAnnotations(groupAnnotations); - return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries, areaGroupRedactions); + return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries); } @@ -376,8 +292,7 @@ public class AnalysisPreparationService { private SectionsToReanalyzeData getDictionaryIncrementAndSectionsToReanalyze(AnalyzeRequest analyzeRequest, DictionaryIncrement dictionaryIncrement, ReanalysisSetupData reanalysisSetupData, - ImportedRedactions importedRedactions, - List groupAnnotations) { + ImportedRedactions importedRedactions) { Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); @@ -388,8 +303,7 @@ public class AnalysisPreparationService { reanalysisSetupData.document(), dictionaryIncrement, importedRedactions, - relevantManuallyModifiedAnnotationIds, - groupAnnotations)); + relevantManuallyModifiedAnnotationIds)); List sectionsToReAnalyse = getSectionsToReAnalyse(reanalysisSetupData.document(), sectionsToReanalyseIds); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -460,31 +374,21 @@ public class AnalysisPreparationService { Document document, DictionaryIncrement dictionaryIncrement, ImportedRedactions importedRedactions, - Set relevantManuallyModifiedAnnotationIds, - List groupAnnotations) { + Set relevantManuallyModifiedAnnotationIds) { - return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, - document, - analyzeRequest, - importedRedactions, - relevantManuallyModifiedAnnotationIds, - groupAnnotations); + return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds); } - private record DictionaryAndNotFoundEntries( - Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries, List areaGroupRedactions - ) { + private record DictionaryAndNotFoundEntries(Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries) { } - private record NotFoundEntries(List notFoundManualRedactionEntries, List notFoundImportedEntries, List areaGroupRedactions) { + private record NotFoundEntries(List notFoundManualRedactionEntries, List notFoundImportedEntries) { } - private record SectionsToReanalyzeData( - DictionaryIncrement dictionaryIncrement, Set sectionsToReanalyseIds, List sectionsToReanalyze - ) { + private record SectionsToReanalyzeData(DictionaryIncrement dictionaryIncrement, Set sectionsToReanalyseIds, List sectionsToReanalyze) { } @@ -496,9 +400,7 @@ public class AnalysisPreparationService { Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries, - NerEntities nerEntities, - GroupAnnotationInternalResponse groupRedactions, - List areaGroupRedactions + NerEntities nerEntities ) { } @@ -515,19 +417,13 @@ public class AnalysisPreparationService { Set sectionsToReanalyseIds, List sectionsToReAnalyse, KieWrapper kieWrapperComponentRules, - KieWrapper kieWrapperEntityRules, - GroupAnnotationInternalResponse groupAnnotations, - List unprocessedAreaGroupAnnotations + KieWrapper kieWrapperEntityRules ) { } public record ReanalysisFinalProcessingData( - NerEntities nerEntities, - Dictionary dictionary, - List notFoundManualRedactionEntries, - List notFoundImportedEntries, - List areaGroupAnnotations + NerEntities nerEntities, Dictionary dictionary, List notFoundManualRedactionEntries, List notFoundImportedEntries ) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 513731e0..d5a4ba19 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -6,9 +6,9 @@ import static com.iqser.red.service.redaction.v1.server.service.AnalysisPreparat import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,11 +24,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.logger.Context; @@ -67,7 +64,6 @@ public class AnalyzeService { FunctionTimerValues redactmanagerAnalyzePagewiseValues; ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE; AnalysisPreparationService analysisPreparationService; - GroupSearchService groupSearchService; @Timed("redactmanager_reanalyze") @@ -92,7 +88,7 @@ public class AnalyzeService { ReanalysisInitialProcessingData initialProcessingData = analysisPreparationService.getReanalysisInitialProcessingData(analyzeRequest, setupData); - if (initialProcessingData.sectionsToReAnalyse().isEmpty() && initialProcessingData.groupAnnotations().getAreaGroupAnnotations().isEmpty()) { + if (initialProcessingData.sectionsToReAnalyse().isEmpty()) { EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(setupData.entityLog(), initialProcessingData.dictionaryIncrement().getDictionaryVersion(), @@ -100,8 +96,6 @@ public class AnalyzeService { new ArrayList<>(), new ArrayList<>()); - Set groupAnnotations = getGroupAnnotations(initialProcessingData.groupAnnotations()); - return finalizeAnalysis(analyzeRequest, startTime, initialProcessingData.kieWrapperComponentRules(), @@ -110,28 +104,17 @@ public class AnalyzeService { setupData.document().getNumberOfPages(), true, new HashSet<>(), - groupAnnotations, context); } context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion()); - ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, - setupData, - initialProcessingData, - initialProcessingData.groupAnnotations()); + ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, setupData, initialProcessingData); - if (!initialProcessingData.sectionsToReAnalyse().isEmpty()) { - dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse()); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse()); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - groupSearchService.addGroupEntities(initialProcessingData.groupAnnotations().getTextGroupAnnotations(), initialProcessingData.sectionsToReAnalyse()); - log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - } - - var notFoundManualOrImportedEntriesOrAreaGroupRedactions = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), - finalProcessingData.notFoundImportedEntries(), - finalProcessingData.areaGroupAnnotations()) + var notFoundManualOrImportedEntries = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), finalProcessingData.notFoundImportedEntries()) .flatMap(Collection::stream) .collect(Collectors.toList()); @@ -149,14 +132,12 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, setupData.document(), setupData.entityLog(), - notFoundManualOrImportedEntriesOrAreaGroupRedactions, + notFoundManualOrImportedEntries, initialProcessingData.sectionsToReanalyseIds(), finalProcessingData.dictionary().getVersion()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries()); - Set groupAnnotations = getGroupAnnotations(initialProcessingData.groupAnnotations()); - return finalizeAnalysis(analyzeRequest, startTime, initialProcessingData.kieWrapperComponentRules(), @@ -165,7 +146,6 @@ public class AnalyzeService { setupData.document().getNumberOfPages(), true, new HashSet<>(allFileAttributes), - groupAnnotations, context); } @@ -186,18 +166,13 @@ public class AnalyzeService { analyzeRequest.getAnalysisNumber(), TenantContext.getTenantId()); - var notFoundManualOrImportedEntriesOrGroupAreaRedaction = Stream.of(analysisData.notFoundManualRedactionEntries(), - analysisData.notFoundImportedEntries(), - analysisData.areaGroupRedactions()) + var notFoundManualOrImportedEntries = Stream.of(analysisData.notFoundManualRedactionEntries(), analysisData.notFoundImportedEntries()) .flatMap(Collection::stream) .collect(Collectors.toList()); dictionarySearchService.addDictionaryEntities(analysisData.dictionary(), analysisData.document()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - groupSearchService.addGroupEntities(analysisData.groupRedactions().getTextGroupAnnotations(), analysisData.document()); - log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - // we could add the imported redactions similar to the manual redactions here as well for additional processing List allFileAttributes = entityDroolsExecutionService.executeRules(analysisData.kieWrapperEntityRules().container(), analysisData.document(), @@ -210,14 +185,12 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, analysisData.document(), - notFoundManualOrImportedEntriesOrGroupAreaRedaction, + notFoundManualOrImportedEntries, analysisData.dictionary().getVersion(), analysisData.kieWrapperEntityRules().rulesVersion()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); - Set groupAnnotations = getGroupAnnotations(analysisData.groupRedactions()); - return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), @@ -226,11 +199,9 @@ public class AnalyzeService { analysisData.document().getNumberOfPages(), false, new HashSet<>(allFileAttributes), - groupAnnotations, context); } - @Timed("redactmanager_analyzeImportedRedactionsOnly") @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly") public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) { @@ -266,17 +237,7 @@ public class AnalyzeService { notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); - Set groupAnnotations = getGroupAnnotations(analysisData.groupRedactions()); - - return finalizeAnalysis(analyzeRequest, - startTime, - analysisData.kieWrapperComponentRules(), - entityLogChanges, - analysisData.document(), - analysisData.document().getNumberOfPages(), - false, - new HashSet<>(), - groupAnnotations, + return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), entityLogChanges, analysisData.document(), analysisData.document().getNumberOfPages(), false, new HashSet<>(), context); } @@ -289,7 +250,6 @@ public class AnalyzeService { int numberOfPages, boolean isReanalysis, Set addedFileAttributes, - Set groupAnnotations, Context context) { EntityLog entityLog = entityLogChanges.getEntityLog(); @@ -335,7 +295,6 @@ public class AnalyzeService { .manualRedactions(analyzeRequest.getManualRedactions()) .addedFileAttributes(addedFileAttributes) .usedComponentMappings(analyzeRequest.getComponentMappings()) - .groupAnnotations(groupAnnotations) .build(); } @@ -370,19 +329,4 @@ public class AnalyzeService { log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } - - private Set getGroupAnnotations(GroupAnnotationInternalResponse groupAnnotationInternalResponse) { - - List areaGroupRedactions = groupAnnotationInternalResponse.getAreaGroupAnnotations(); - List textGroupRedactions = groupAnnotationInternalResponse.getTextGroupAnnotations(); - - return Stream.concat(Optional.ofNullable(areaGroupRedactions) - .orElseGet(List::of) - .stream(), - Optional.ofNullable(textGroupRedactions) - .orElseGet(List::of) - .stream()) - .collect(Collectors.toSet()); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index fb8800f7..e1be0b90 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -15,7 +15,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange; @@ -45,25 +44,10 @@ public class EntityChangeLogService { List toInsert = new ArrayList<>(); List toUpdate = new ArrayList<>(); - List existingAreaGroupEntries = new ArrayList<>(); - for (EntityLogEntry entityLogEntry : newEntityLogEntries) { Optional optionalPreviousEntity = previousEntityLogEntries.stream() .filter(entry -> entry.getId().equals(entityLogEntry.getId())) .findAny(); - - Optional optionalPreviousAreaGroupEntity = previousEntityLogEntries.stream() - .filter(entry -> entry.getGroupId() != null && entry.getGroupId().equals(entityLogEntry.getGroupId())) - .filter(entry -> entry.getEntryType().equals(EntryType.AREA)) - .filter(entry -> entry.getPositions().equals(entityLogEntry.getPositions())) - .findAny(); - - // Skip group area entities on the same position - if (optionalPreviousAreaGroupEntity.isPresent()) { - existingAreaGroupEntries.add(optionalPreviousAreaGroupEntity.get()); - continue; - } - if (optionalPreviousEntity.isEmpty()) { entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())); toInsert.add(entityLogEntry); @@ -81,21 +65,19 @@ public class EntityChangeLogService { } } - toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, existingAreaGroupEntries, analysisNumber, now)); + toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now)); return new EntryChanges(toInsert, toUpdate); } private List addRemovedEntriesAsRemoved(List previousEntityLogEntries, - List newEntityLogEntries, List existingAreaGroupEntries, int analysisNumber, + List newEntityLogEntries, + int analysisNumber, OffsetDateTime now) { Set existingIds = newEntityLogEntries.stream() .map(EntityLogEntry::getId) .collect(Collectors.toSet()); - existingIds.addAll(existingAreaGroupEntries.stream() - .map(EntityLogEntry::getId) - .collect(Collectors.toSet())); List removedEntries = previousEntityLogEntries.stream() .filter(entry -> !existingIds.contains(entry.getId())) .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index f1bec55c..182b93f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -238,7 +238,7 @@ public class EntityLogCreatorService { List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); List allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); - EntityLogEntry entityLogEntry = EntityLogEntry.builder() + return EntityLogEntry.builder() .id(precursorEntity.getId()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(precursorEntity.legalBasis()) @@ -267,12 +267,9 @@ public class EntityLogCreatorService { //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) .imported(false) .reference(Collections.emptySet()) - .manualChanges(precursorEntity.getGroupId() == null ? ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges) : Collections.emptyList()) - .groupId(precursorEntity.getGroupId()) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) .paragraphPageIdx(-1) .build(); - - return entityLogEntry; } @@ -321,7 +318,6 @@ public class EntityLogCreatorService { .state(buildEntryState(entity)) .entryType(entryType) .paragraphPageIdx(determinePageParagraphIndex(entity, entryType)) - .groupId(entity.getGroupId()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java deleted file mode 100644 index 2e585cf3..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.service; - -import java.util.List; -import java.util.Set; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; - -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class GroupSearchService { - - EntityEnrichmentService entityEnrichmentService; - - - public void addGroupEntities(List groupAnnotations, List nodes) { - - nodes.forEach(node -> addGroupEntities(groupAnnotations, node)); - } - - - public void addGroupEntities(List groupAnnotations, SemanticNode node) { - - groupAnnotations.stream() - .filter(groupRedaction -> groupRedaction.getSoftDeletedTime() == null) - .forEach(groupRedaction -> { - SearchImplementation searchImplementation = new SearchImplementation(groupRedaction.getValue(), false); - bySearchImplementationAsGroup(searchImplementation, - groupRedaction.getTypeId(), - calculateEntityType(groupRedaction.getEntryType()), - groupRedaction.getGroupId(), - groupRedaction.getLegalBasis(), - node); - }); - } - - - public void bySearchImplementationAsGroup(SearchImplementation searchImplementation, String type, EntityType entityType, String groupId, String legalBasis, SemanticNode node) { - - Set engines = Set.of(Engine.GROUP); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) - .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) - .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines) - .ifPresent(entity -> { - if (entity.getEntityType().equals(EntityType.HINT)) { - entity.skip("GRP.0.1", "group hint is skipped by default"); - } else { - entity.apply("GRP.0.0", "group entries are applied by default", legalBasis); - } - entity.setGroupId(groupId); - })); - } - - - private EntityType calculateEntityType(EntryType entryType) { - - switch (entryType) { - case FALSE_RECOMMENDATION -> { - return EntityType.FALSE_RECOMMENDATION; - } - case RECOMMENDATION -> { - return EntityType.RECOMMENDATION; - } - case HINT -> { - return EntityType.HINT; - } - case FALSE_POSITIVE -> { - return EntityType.FALSE_POSITIVE; - } - default -> { - return EntityType.ENTITY; - } - } - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupAnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupAnnotationService.java deleted file mode 100644 index 8c95e561..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupAnnotationService.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.service.document; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation; -import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class AreaGroupAnnotationService { - - public List createAreaGroupAnnotations(List groupAnnotations) { - - List precursorEntities = new ArrayList<>(); - if (groupAnnotations != null) { - groupAnnotations.forEach(groupAnnotation -> precursorEntities.addAll(PrecursorEntity.fromAreaGroupAnnotation(groupAnnotation))); - } - return precursorEntities; - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index bcda5c61..f45a3299 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -11,6 +11,7 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; @@ -22,14 +23,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationType; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -48,29 +46,20 @@ public class SectionFinderService { Document document, AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, - Set relevantManuallyModifiedAnnotationIds, - List groupAnnotations) { + Set relevantManuallyModifiedAnnotationIds) { long start = System.currentTimeMillis(); Set sectionsToReanalyse = new HashSet<>(); + var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() .stream() .map(DictionaryIncrementValue::getValue) .toList(), true); - List unprocessedGroupRedactions = groupAnnotations.stream() - .filter(groupAnnotation -> groupAnnotation.getGroupAnnotationType().equals(GroupAnnotationType.TEXT)) - .filter(UnprocessedUtils::isUnprocessedGroupAnnotation) - .map(GroupAnnotation::getValue) - .toList(); - - var groupRedactionSearch = new SearchImplementation(unprocessedGroupRedactions, false); - document.streamChildren() .forEach(mainNode -> { - if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText()) || groupRedactionSearch.atLeastOneMatches(mainNode.getTextBlock() - .getSearchText())) { + if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) { sectionsToReanalyse.add(mainNode.getTreeId() .get(0)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java deleted file mode 100644 index 1d5b95ef..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.utils; - -import java.time.OffsetDateTime; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupChange; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class UnprocessedUtils { - - public static boolean isUnprocessedGroupAnnotation(GroupAnnotation groupRedaction) { - - if (groupRedaction.getGroupChanges().isEmpty() && groupRedaction.getProcessedDate() == null) { - return true; - } - - if (!groupRedaction.getGroupChanges().isEmpty()) { - OffsetDateTime processedDate = groupRedaction.getProcessedDate(); - - if (processedDate == null) { - return true; - } - - OffsetDateTime lastChangeDate = groupRedaction.getGroupChanges() - .stream() - .map(GroupChange::getChangeDate) - .max(OffsetDateTime::compareTo) - .orElse(null); - - return lastChangeDate != null && lastChangeDate.isAfter(processedDate); - } - - return false; - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index ec3430bf..8d15894b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -51,7 +51,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository. import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; -import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; @@ -212,9 +211,6 @@ public abstract class AbstractRedactionIntegrationTest { @MockBean protected DictionaryClient dictionaryClient; - @MockBean - protected GroupRedactionClient groupRedactionClient; - @MockBean protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index 5fa81672..7b0cc6bc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -5,8 +5,6 @@ import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTe import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME; import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import java.io.File; @@ -14,7 +12,6 @@ import java.io.FileInputStream; import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -51,9 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; -import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; @@ -106,8 +101,6 @@ import lombok.extern.slf4j.Slf4j; @MockBean protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver; @MockBean - protected GroupRedactionClient groupRedactionClient; - @MockBean private RedisSyncedWebSocketService redisSyncedWebSocketService; @MockBean private RedisMessageListenerContainer redisPubsubContainer; @@ -188,10 +181,6 @@ import lombok.extern.slf4j.Slf4j; String type = invocation.getArgument(0); return testDossierTemplate.testDictionary.isHint(type); }); - when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder() - .textGroupAnnotations(new ArrayList<>()) - .areaGroupAnnotations(new ArrayList<>()) - .build()); when(dictionaryService.getColor(any(String.class), any())).thenAnswer(invocation -> { String type = invocation.getArgument(0); return testDossierTemplate.testDictionary.getType(type).getColor(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 77e77dc6..d96438fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -3,8 +3,7 @@ package com.iqser.red.service.redaction.v1.server; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -41,7 +40,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; @@ -53,8 +52,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -import edu.emory.mathcs.backport.java.util.Collections; - @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"}) @Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class) @@ -232,11 +229,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); - - when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder() - .textGroupAnnotations(Collections.emptyList()) - .areaGroupAnnotations(Collections.emptyList()) - .build()); } @@ -282,7 +274,9 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { .fileId(TEST_FILE_ID) .build()); }); - request.setManualRedactions(ManualRedactions.builder().idsToRemove(idRemovalSet).build()); + request.setManualRedactions(ManualRedactions.builder() + .idsToRemove(idRemovalSet) + .build()); analyzeService.reanalyze(request); @@ -310,20 +304,12 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var doseMortality = componentLog.getComponentLogEntries() - .stream() - .filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality")) - .findFirst() - .get(); + var doseMortality = componentLog.getComponentLogEntries().stream().filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality")).findFirst().get(); assertEquals(doseMortality.getComponentValues().size(), 5); Pattern pattern = Pattern.compile("^5000, [SD]$"); - boolean allMatch = doseMortality.getComponentValues() - .stream() - .map(ComponentLogEntryValue::getValue) - .allMatch(pattern.asPredicate()); + boolean allMatch = doseMortality.getComponentValues().stream().map(ComponentLogEntryValue::getValue).allMatch(pattern.asPredicate()); assertTrue(allMatch); } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java deleted file mode 100644 index 0f469eca..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java +++ /dev/null @@ -1,158 +0,0 @@ -package com.iqser.red.service.redaction.v1.server; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.when; - -import java.io.FileOutputStream; -import java.util.List; -import java.util.UUID; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import com.iqser.red.commons.jackson.ObjectMapperFactory; -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupAnnotation; -import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; -import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; -import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; -import com.knecon.fforesight.tenantcommons.TenantContext; - -import lombok.SneakyThrows; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Import(GroupRedactionTest.GroupRedactionTestConfiguration.class) -public class GroupRedactionTest extends AbstractRedactionIntegrationTest { - - private static final String RULES = loadFromClassPath("drools/rules.drl"); - - @Configuration - @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @Import({LayoutParsingServiceProcessorConfiguration.class}) - @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) - public static class GroupRedactionTestConfiguration { - - @Bean - @Primary - public StorageService inmemoryStorage() { - - return new FileSystemBackedStorageService(ObjectMapperFactory.create()); - } - - } - - - @BeforeEach - public void stubClients() { - - TenantContext.setTenantId("redaction"); - - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L); - - loadDictionaryForTest(); - loadTypeForTest(); - loadNerForTest(); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse()); - - when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L); - when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); - - mockDictionaryCalls(null); - - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); - } - - - @Test - @SneakyThrows - public void testGroupRedaction() { - - String textID = UUID.randomUUID().toString(); - String areaID = UUID.randomUUID().toString(); - - TextGroupAnnotation textGroupRedaction = TextGroupAnnotation.builder() - .positionOnPage(PositionOnPage.builder().x(423.9f).y(505.5f).width(43.9f).height(15.4f).pageNumber(1).build()) - .groupId(textID) - .typeId("CBI_author") - .value("medicine") - .legalBasis("test legal basis") - .entryType(EntryType.ENTITY) - .groupAnnotationType(GroupAnnotationType.TEXT) - .build(); - - AreaGroupAnnotation areaGroupRedaction = AreaGroupAnnotation.builder() - .positionOnPage(PositionOnPage.builder().x(346.27f).y(644.24f).width(219.4f).height(52.24f).pageNumber(2).build()) - .groupId(areaID) - .pageRanges(List.of(PageRange.builder().startPage(2).endPage(5).build())) - .typeId("PII") - .value("non-readable content") - .entryType(EntryType.AREA) - .groupAnnotationType(GroupAnnotationType.AREA) - .section("my section") - .build(); - - when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder() - .textGroupAnnotations(List.of(textGroupRedaction)) - .areaGroupAnnotations(List.of(areaGroupRedaction)) - .build()); - - AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf"); - System.out.println("Start Full integration test"); - analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); - System.out.println("Finished structure analysis"); - analyzeService.analyze(request); - System.out.println("Finished analysis"); - var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); - - String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; - - try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { - fileOutputStream.write(annotateResponse.getDocument()); - } - - var textGroupRedactions = entityLog.getEntityLogEntry() - .stream() - .filter(e -> e.getGroupId() != null && e.getGroupId().equals(textID)) - .toList(); - assertEquals(textGroupRedactions.size(), 6); - - var areaGroupRedactions = entityLog.getEntityLogEntry() - .stream() - .filter(e -> e.getGroupId() != null && e.getGroupId().equals(areaID)) - .toList(); - assertEquals(areaGroupRedactions.size(), 4); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index dafca18b..c7dad2bc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -47,7 +45,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; @@ -60,7 +58,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -import edu.emory.mathcs.backport.java.util.Collections; import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @@ -109,11 +106,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { mockDictionaryCalls(null); - when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder() - .textGroupAnnotations(Collections.emptyList()) - .areaGroupAnnotations(Collections.emptyList()) - .build()); - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 414def3b..ac368c12 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; @@ -68,7 +67,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; @@ -85,7 +83,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -import edu.emory.mathcs.backport.java.util.Collections; import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @@ -132,11 +129,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { mockDictionaryCalls(null); - when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder() - .textGroupAnnotations(Collections.emptyList()) - .areaGroupAnnotations(Collections.emptyList()) - .build()); - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index c00a9e2d..84ffd1f0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -3,8 +3,7 @@ package com.iqser.red.service.redaction.v1.server.service.document; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -42,7 +41,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupAnnotationInternalResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; @@ -56,7 +55,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; -import edu.emory.mathcs.backport.java.util.Collections; import lombok.SneakyThrows; @ExtendWith(SpringExtension.class) @@ -107,11 +105,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse()); mockDictionaryCalls(null); - when(groupRedactionClient.getGroupAnnotations(anyString(), anyString(), anyBoolean())).thenReturn(GroupAnnotationInternalResponse.builder() - .textGroupAnnotations(Collections.emptyList()) - .areaGroupAnnotations(Collections.emptyList()) - .build()); - when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); }