Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7327' into 'master'

Browse Source 
RED-7327 - Add group redactions

Closes RED-7327

See merge request redactmanager/redaction-service!486
This commit is contained in:
Andrei Isvoran 2024-08-20 10:32:09 +02:00
commit 78990b5555
22 changed files with 661 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -4,7 +4,7 @@ plugins {
}
description = "redaction-service-api-v1"
val persistenceServiceVersion = "2.504.0"
val persistenceServiceVersion = "2.522.0"
dependencies {
implementation("org.springframework:spring-web:6.0.12")

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.141.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.509.0"
val persistenceServiceVersion = "2.522.0"
val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4"
val testContainersVersion = "1.19.7"

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/GroupRedactionClient.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.redaction.v1.server.client;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.internal.resources.GroupRedactionResource;
@FeignClient(name = "GroupRedactionResource", url = "${persistence-service.url}")
public interface GroupRedactionClient extends GroupRedactionResource {
}

47
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model;
import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.PriorityQueue;
@ -14,11 +15,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@ -48,6 +51,7 @@ public class PrecursorEntity implements IEntity {
boolean isDossierDictionaryEntry;
boolean rectangle;
Set<Engine> engines;
String groupId;
@Builder.Default
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
@ -149,6 +153,49 @@ public class PrecursorEntity implements IEntity {
}
public static List<PrecursorEntity> fromAreaGroupRedaction(AreaGroupRedaction areaGroupRedaction) {
List<PrecursorEntity> precursorEntities = new ArrayList<>();
areaGroupRedaction.getPageRanges()
.forEach(pageRange -> {
for (int pageStart = pageRange.getStartPage(); pageStart <= pageRange.getEndPage(); pageStart++) {
boolean isHint = areaGroupRedaction.getEntryType() == EntryType.HINT;
String ruleIdentifier = isHint ? "GRP.5.0" : "GRP.5.1";
String reason = isHint ? "group hint is skipped by default" : "group area is applied by default";
String legalBasis = isHint ? "" : areaGroupRedaction.getLegalBasis();
MatchedRule.MatchedRuleBuilder matchedRule = MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason);
if (!isHint) {
matchedRule.legalBasis(legalBasis);
matchedRule.applied(true);
}
precursorEntities.add(PrecursorEntity.builder()
.id(UUID.randomUUID().toString())
.value(areaGroupRedaction.getValue())
.entityPosition(List.of(RectangleWithPage.fromPositionOnPage(pageStart, areaGroupRedaction.getPositionOnPage())))
.ruleIdentifier(ruleIdentifier)
.matchedRuleList(new PriorityQueue<>(List.of(matchedRule.build())))
.reason(reason)
.legalBasis(legalBasis)
.section(areaGroupRedaction.getSection())
.type(areaGroupRedaction.getTypeId())
.entityType(isHint ? EntityType.HINT : EntityType.ENTITY)
.applied(!isHint)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
.rectangle(true)
.section(areaGroupRedaction.getSection())
.engines(Set.of(Engine.GROUP))
.groupId(areaGroupRedaction.getGroupId())
.manualOverwrite(new ManualChangeOverwrite(isHint ? EntityType.HINT : EntityType.ENTITY))
.build());
}
});
return precursorEntities;
}
public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.model;
import java.awt.geom.Rectangle2D;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
@ -19,6 +20,12 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
}
public static RectangleWithPage fromPositionOnPage(int pageNumber, PositionOnPage positionOnPage) {
return new RectangleWithPage(pageNumber, toRectangle2D(positionOnPage));
}
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
@ -42,4 +49,10 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
}
private static Rectangle2D toRectangle2D(PositionOnPage positionOnPage) {
return new Rectangle2D.Float(positionOnPage.getX(), positionOnPage.getY(), positionOnPage.getWidth(), positionOnPage.getHeight());
}
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Setter;
import lombok.experimental.FieldDefaults;
@Builder

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -29,7 +29,9 @@ public final class MatchedRule implements Comparable<MatchedRule> {
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN");
public static final RuleType GROUP_TYPE = RuleType.fromString("GRP");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE, MANUAL_TYPE, GROUP_TYPE);
RuleIdentifier ruleIdentifier;
@Builder.Default

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -49,6 +49,7 @@ public class TextEntity implements IEntity {
boolean dictionaryEntry;
boolean dossierDictionaryEntry;
String groupId;
@Builder.Default
Set<Engine> engines = new HashSet<>();

141
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -6,9 +6,11 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.task.TaskExecutor;
@ -18,7 +20,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
@ -28,6 +36,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.AreaGroupRedactionService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
@ -36,6 +45,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils;
import lombok.AccessLevel;
import lombok.SneakyThrows;
@ -55,7 +65,9 @@ public class AnalysisPreparationService {
ImportedRedactionEntryService importedRedactionEntryService;
DictionaryService dictionaryService;
SectionFinderService sectionFinderService;
GroupRedactionClient groupRedactionClient;
TaskExecutor taskExecutor;
AreaGroupRedactionService areaGroupRedactionService;
public AnalysisPreparationService(KieContainerCreationService kieContainerCreationService,
@ -66,7 +78,9 @@ public class AnalysisPreparationService {
ImportedRedactionEntryService importedRedactionEntryService,
DictionaryService dictionaryService,
SectionFinderService sectionFinderService,
@Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) {
GroupRedactionClient groupRedactionClient,
@Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor,
AreaGroupRedactionService areaGroupRedactionService) {
this.kieContainerCreationService = kieContainerCreationService;
this.observedStorageService = observedStorageService;
@ -76,7 +90,9 @@ public class AnalysisPreparationService {
this.importedRedactionEntryService = importedRedactionEntryService;
this.dictionaryService = dictionaryService;
this.sectionFinderService = sectionFinderService;
this.groupRedactionClient = groupRedactionClient;
this.taskExecutor = taskExecutor;
this.areaGroupRedactionService = areaGroupRedactionService;
}
@ -93,6 +109,8 @@ public class AnalysisPreparationService {
CompletableFuture<NerEntities> nerEntitiesFuture = documentFuture.thenApplyAsync((document) -> getNerEntities(analyzeRequest, document), taskExecutor);
CompletableFuture<GroupRedactionInternalResponse> internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupRedactions(analyzeRequest, false), taskExecutor);
CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join();
Dictionary dictionary = getDictionary(analyzeRequest);
@ -106,6 +124,8 @@ public class AnalysisPreparationService {
List<PrecursorEntity> notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
List<PrecursorEntity> areaGroupRedactions = areaGroupRedactionService.createAreaGroupRedactions(internalResponseFuture.get().getAreaGroupRedactions());
return new AnalysisData(kieWrapperEntityRulesFuture.get(),
kieWrapperComponentRulesFuture.get(),
document,
@ -113,7 +133,9 @@ public class AnalysisPreparationService {
dictionary,
notFoundManualRedactionEntries,
notFoundImportedEntries,
nerEntitiesFuture.get());
nerEntitiesFuture.get(),
internalResponseFuture.get(),
areaGroupRedactions);
}
@ -135,11 +157,35 @@ public class AnalysisPreparationService {
CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
CompletableFuture<SectionsToReanalyzeData> incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenApplyAsync((importedRedactions) -> {
DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(analyzeRequest, reanalysisSetupData);
return getDictionaryIncrementAndSectionsToReanalyze(analyzeRequest, dictionaryIncrement, reanalysisSetupData, importedRedactions);
CompletableFuture<GroupRedactionInternalResponse> internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupRedactions(analyzeRequest, true), taskExecutor);
}, taskExecutor);
CompletableFuture<SectionsToReanalyzeData> incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenCombineAsync(internalResponseFuture,
(importedRedactions, groupRedactions) -> {
List<AreaGroupRedaction> areaGroupRedactions = groupRedactions.getAreaGroupRedactions();
List<TextGroupRedaction> textGroupRedactions = groupRedactions.getTextGroupRedactions();
List<GroupRedaction> internalGroupRedactions = Stream.concat(
Optional.ofNullable(
areaGroupRedactions)
.orElseGet(List::of)
.stream(),
Optional.ofNullable(
textGroupRedactions)
.orElseGet(List::of)
.stream())
.toList();
DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(
analyzeRequest,
reanalysisSetupData);
return getDictionaryIncrementAndSectionsToReanalyze(
analyzeRequest,
dictionaryIncrement,
reanalysisSetupData,
importedRedactions,
internalGroupRedactions);
},
taskExecutor);
CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
@ -147,19 +193,26 @@ public class AnalysisPreparationService {
CompletableFuture.allOf(importedRedactionsFuture, incrementAndSectionsToReanalyzeFuture, kieWrapperComponentRulesFuture, kieWrapperEntityRulesFuture).join();
List<AreaGroupRedaction> areaGroupRedactions = internalResponseFuture.get().getAreaGroupRedactions().stream()
.filter(UnprocessedUtils::isUnprocessedGroupedRedaction)
.toList();
return new ReanalysisInitialProcessingData(importedRedactionsFuture.get(),
incrementAndSectionsToReanalyzeFuture.get().dictionaryIncrement(),
incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyseIds(),
incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyze(),
kieWrapperComponentRulesFuture.get(),
kieWrapperEntityRulesFuture.get());
kieWrapperEntityRulesFuture.get(),
internalResponseFuture.get(),
areaGroupRedactions);
}
@SneakyThrows
public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest,
ReanalysisSetupData reanalysisSetupData,
ReanalysisInitialProcessingData reanalysisInitialProcessingData) {
ReanalysisInitialProcessingData reanalysisInitialProcessingData,
GroupRedactionInternalResponse groupRedactions) {
CompletableFuture<NerEntities> nerEntitiesFuture = CompletableFuture.supplyAsync(() -> getNerEntitiesFiltered(analyzeRequest,
reanalysisSetupData.document,
@ -168,16 +221,25 @@ public class AnalysisPreparationService {
CompletableFuture<DictionaryAndNotFoundEntries> dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> {
Dictionary dictionary = getDictionary(analyzeRequest);
NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions());
return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries());
NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest,
reanalysisSetupData.document(),
reanalysisInitialProcessingData.importedRedactions(),
groupRedactions.getAreaGroupRedactions().stream().filter(gR -> gR.getSoftDeletedTime() == null).toList());
return new DictionaryAndNotFoundEntries(dictionary,
notFoundEntries.notFoundManualRedactionEntries(),
notFoundEntries.notFoundImportedEntries(),
notFoundEntries.areaGroupRedactions());
}, taskExecutor);
CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture).join();
CompletableFuture<GroupRedactionInternalResponse> internalResponseFuture = CompletableFuture.supplyAsync(() -> getGroupRedactions(analyzeRequest, false), taskExecutor);
CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture, internalResponseFuture).join();
return new ReanalysisFinalProcessingData(nerEntitiesFuture.get(),
dictionaryAndNotFoundEntriesCompletableFuture.get().dictionary(),
dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundManualRedactionEntries(),
dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries());
dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries(),
dictionaryAndNotFoundEntriesCompletableFuture.get().areaGroupRedactions());
}
@ -201,6 +263,14 @@ public class AnalysisPreparationService {
}
public GroupRedactionInternalResponse getGroupRedactions(AnalyzeRequest analyzeRequest, boolean includeDeleted) {
GroupRedactionInternalResponse groupRedactionResponse = groupRedactionClient.getGroupRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), includeDeleted);
log.info("Loaded group redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
return groupRedactionResponse;
}
public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
@ -234,13 +304,14 @@ public class AnalysisPreparationService {
}
private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) {
private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions, List<AreaGroupRedaction> groupRedactions) {
var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries);
var areaGroupRedactions = areaGroupRedactionService.createAreaGroupRedactions(groupRedactions);
return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries, areaGroupRedactions);
}
@ -267,7 +338,8 @@ public class AnalysisPreparationService {
private SectionsToReanalyzeData getDictionaryIncrementAndSectionsToReanalyze(AnalyzeRequest analyzeRequest,
DictionaryIncrement dictionaryIncrement,
ReanalysisSetupData reanalysisSetupData,
ImportedRedactions importedRedactions) {
ImportedRedactions importedRedactions,
List<GroupRedaction> groupRedactions) {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@ -278,7 +350,8 @@ public class AnalysisPreparationService {
reanalysisSetupData.document(),
dictionaryIncrement,
importedRedactions,
relevantManuallyModifiedAnnotationIds));
relevantManuallyModifiedAnnotationIds,
groupRedactions));
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(reanalysisSetupData.document(), sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -339,21 +412,33 @@ public class AnalysisPreparationService {
Document document,
DictionaryIncrement dictionaryIncrement,
ImportedRedactions importedRedactions,
Set<String> relevantManuallyModifiedAnnotationIds) {
Set<String> relevantManuallyModifiedAnnotationIds,
List<GroupRedaction> groupRedactions) {
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds);
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
document,
analyzeRequest,
importedRedactions,
relevantManuallyModifiedAnnotationIds,
groupRedactions);
}
private record DictionaryAndNotFoundEntries(Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries) {
private record DictionaryAndNotFoundEntries(
Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries, List<PrecursorEntity> areaGroupRedactions
) {
}
private record NotFoundEntries(List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries) {
private record NotFoundEntries(List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries, List<PrecursorEntity> areaGroupRedactions) {
}
private record SectionsToReanalyzeData(DictionaryIncrement dictionaryIncrement, Set<Integer> sectionsToReanalyseIds, List<SemanticNode> sectionsToReanalyze) {
private record SectionsToReanalyzeData(
DictionaryIncrement dictionaryIncrement,
Set<Integer> sectionsToReanalyseIds,
List<SemanticNode> sectionsToReanalyze
) {
}
@ -365,7 +450,9 @@ public class AnalysisPreparationService {
Dictionary dictionary,
List<PrecursorEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundImportedEntries,
NerEntities nerEntities
NerEntities nerEntities,
GroupRedactionInternalResponse groupRedactions,
List<PrecursorEntity> areaGroupRedactions
) {
}
@ -382,13 +469,19 @@ public class AnalysisPreparationService {
Set<Integer> sectionsToReanalyseIds,
List<SemanticNode> sectionsToReAnalyse,
KieWrapper kieWrapperComponentRules,
KieWrapper kieWrapperEntityRules
KieWrapper kieWrapperEntityRules,
GroupRedactionInternalResponse groupRedactions,
List<AreaGroupRedaction> unprocessedAreaGroupRedactions
) {
}
public record ReanalysisFinalProcessingData(
NerEntities nerEntities, Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries
NerEntities nerEntities,
Dictionary dictionary,
List<PrecursorEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundImportedEntries,
List<PrecursorEntity> areaGroupRedactions
) {
}

77
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -9,6 +9,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -24,8 +25,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context;
@ -64,6 +68,7 @@ public class AnalyzeService {
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;
AnalysisPreparationService analysisPreparationService;
GroupSearchService groupSearchService;
@Timed("redactmanager_reanalyze")
@ -88,7 +93,7 @@ public class AnalyzeService {
ReanalysisInitialProcessingData initialProcessingData = analysisPreparationService.getReanalysisInitialProcessingData(analyzeRequest, setupData);
if (initialProcessingData.sectionsToReAnalyse().isEmpty()) {
if (initialProcessingData.sectionsToReAnalyse().isEmpty() && initialProcessingData.groupRedactions().getAreaGroupRedactions().isEmpty()) {
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(setupData.entityLog(),
initialProcessingData.dictionaryIncrement().getDictionaryVersion(),
@ -96,6 +101,8 @@ public class AnalyzeService {
new ArrayList<>(),
new ArrayList<>());
Set<GroupRedaction> groupRedactions = getGroupRedactionsList(initialProcessingData.groupRedactions());
return finalizeAnalysis(analyzeRequest,
startTime,
initialProcessingData.kieWrapperComponentRules(),
@ -104,17 +111,28 @@ public class AnalyzeService {
setupData.document().getNumberOfPages(),
true,
new HashSet<>(),
groupRedactions,
context);
}
context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion());
ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, setupData, initialProcessingData);
ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest,
setupData,
initialProcessingData,
initialProcessingData.groupRedactions());
dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse());
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (!initialProcessingData.sectionsToReAnalyse().isEmpty()) {
dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse());
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var notFoundManualOrImportedEntries = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), finalProcessingData.notFoundImportedEntries())
groupSearchService.addGroupEntities(initialProcessingData.groupRedactions().getTextGroupRedactions(), initialProcessingData.sectionsToReAnalyse());
log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
var notFoundManualOrImportedEntriesOrAreaGroupRedactions = Stream.of(finalProcessingData.notFoundManualRedactionEntries(),
finalProcessingData.notFoundImportedEntries(),
finalProcessingData.areaGroupRedactions())
.flatMap(Collection::stream)
.collect(Collectors.toList());
@ -132,12 +150,14 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
setupData.document(),
setupData.entityLog(),
notFoundManualOrImportedEntries,
notFoundManualOrImportedEntriesOrAreaGroupRedactions,
initialProcessingData.sectionsToReanalyseIds(),
finalProcessingData.dictionary().getVersion());
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries());
Set<GroupRedaction> groupRedactions = getGroupRedactionsList(initialProcessingData.groupRedactions());
return finalizeAnalysis(analyzeRequest,
startTime,
initialProcessingData.kieWrapperComponentRules(),
@ -146,6 +166,7 @@ public class AnalyzeService {
setupData.document().getNumberOfPages(),
true,
new HashSet<>(allFileAttributes),
groupRedactions,
context);
}
@ -166,13 +187,18 @@ public class AnalyzeService {
analyzeRequest.getAnalysisNumber(),
TenantContext.getTenantId());
var notFoundManualOrImportedEntries = Stream.of(analysisData.notFoundManualRedactionEntries(), analysisData.notFoundImportedEntries())
var notFoundManualOrImportedEntriesOrGroupAreaRedaction = Stream.of(analysisData.notFoundManualRedactionEntries(),
analysisData.notFoundImportedEntries(),
analysisData.areaGroupRedactions())
.flatMap(Collection::stream)
.collect(Collectors.toList());
dictionarySearchService.addDictionaryEntities(analysisData.dictionary(), analysisData.document());
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
groupSearchService.addGroupEntities(analysisData.groupRedactions().getTextGroupRedactions(), analysisData.document());
log.info("Finished Group redaction search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(analysisData.kieWrapperEntityRules().container(),
analysisData.document(),
@ -185,12 +211,14 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
analysisData.document(),
notFoundManualOrImportedEntries,
notFoundManualOrImportedEntriesOrGroupAreaRedaction,
analysisData.dictionary().getVersion(),
analysisData.kieWrapperEntityRules().rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
Set<GroupRedaction> groupRedactions = getGroupRedactionsList(analysisData.groupRedactions());
return finalizeAnalysis(analyzeRequest,
startTime,
analysisData.kieWrapperComponentRules(),
@ -199,9 +227,11 @@ public class AnalyzeService {
analysisData.document().getNumberOfPages(),
false,
new HashSet<>(allFileAttributes),
groupRedactions,
context);
}
@Timed("redactmanager_analyzeImportedRedactionsOnly")
@Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly")
public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) {
@ -237,7 +267,17 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
return finalizeAnalysis(analyzeRequest, startTime, analysisData.kieWrapperComponentRules(), entityLogChanges, analysisData.document(), analysisData.document().getNumberOfPages(), false, new HashSet<>(),
Set<GroupRedaction> groupRedactions = getGroupRedactionsList(analysisData.groupRedactions());
return finalizeAnalysis(analyzeRequest,
startTime,
analysisData.kieWrapperComponentRules(),
entityLogChanges,
analysisData.document(),
analysisData.document().getNumberOfPages(),
false,
new HashSet<>(),
groupRedactions,
context);
}
@ -250,6 +290,7 @@ public class AnalyzeService {
int numberOfPages,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes,
Set<GroupRedaction> groupRedactions,
Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog();
@ -295,6 +336,7 @@ public class AnalyzeService {
.manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes)
.usedComponentMappings(analyzeRequest.getComponentMappings())
.groupRedactions(groupRedactions)
.build();
}
@ -329,4 +371,19 @@ public class AnalyzeService {
log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
private Set<GroupRedaction> getGroupRedactionsList(GroupRedactionInternalResponse groupRedactionInternalResponse) {
List<AreaGroupRedaction> areaGroupRedactions = groupRedactionInternalResponse.getAreaGroupRedactions();
List<TextGroupRedaction> textGroupRedactions = groupRedactionInternalResponse.getTextGroupRedactions();
return Stream.concat(Optional.ofNullable(areaGroupRedactions)
.orElseGet(List::of)
.stream(),
Optional.ofNullable(textGroupRedactions)
.orElseGet(List::of)
.stream())
.collect(Collectors.toSet());
}
}

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
@ -44,10 +45,25 @@ public class EntityChangeLogService {
List<EntityLogEntry> toInsert = new ArrayList<>();
List<EntityLogEntry> toUpdate = new ArrayList<>();
List<EntityLogEntry> existingAreaGroupEntries = new ArrayList<>();
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.findAny();
Optional<EntityLogEntry> optionalPreviousAreaGroupEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getGroupId() != null && entry.getGroupId().equals(entityLogEntry.getGroupId()))
.filter(entry -> entry.getEntryType().equals(EntryType.AREA))
.filter(entry -> entry.getPositions().equals(entityLogEntry.getPositions()))
.findAny();
// Skip group area entities on the same position
if (optionalPreviousAreaGroupEntity.isPresent()) {
existingAreaGroupEntries.add(optionalPreviousAreaGroupEntity.get());
continue;
}
if (optionalPreviousEntity.isEmpty()) {
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()));
toInsert.add(entityLogEntry);
@ -65,19 +81,21 @@ public class EntityChangeLogService {
}
}
toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now));
toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, existingAreaGroupEntries, analysisNumber, now));
return new EntryChanges(toInsert, toUpdate);
}
private List<EntityLogEntry> addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
List<EntityLogEntry> newEntityLogEntries,
int analysisNumber,
List<EntityLogEntry> newEntityLogEntries, List<EntityLogEntry> existingAreaGroupEntries, int analysisNumber,
OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
existingIds.addAll(existingAreaGroupEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet()));
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !existingIds.contains(entry.getId()))
.toList();

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@ -238,7 +239,7 @@ public class EntityLogCreatorService {
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
return EntityLogEntry.builder()
EntityLogEntry entityLogEntry = EntityLogEntry.builder()
.id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
@ -267,9 +268,12 @@ public class EntityLogCreatorService {
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
.reference(Collections.emptySet())
.manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
.manualChanges(precursorEntity.getGroupId() == null ? ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges) : Collections.emptyList())
.groupId(precursorEntity.getGroupId())
.paragraphPageIdx(-1)
.build();
return entityLogEntry;
}
@ -318,6 +322,7 @@ public class EntityLogCreatorService {
.state(buildEntryState(entity))
.entryType(entryType)
.paragraphPageIdx(determinePageParagraphIndex(entity, entryType))
.groupId(entity.getGroupId())
.build();
}

92
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/GroupSearchService.java Normal file
View File

@ -0,0 +1,92 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class GroupSearchService {
EntityEnrichmentService entityEnrichmentService;
public void addGroupEntities(List<TextGroupRedaction> groupRedactions, List<SemanticNode> nodes) {
nodes.forEach(node -> addGroupEntities(groupRedactions, node));
}
public void addGroupEntities(List<TextGroupRedaction> groupRedactions, SemanticNode node) {
groupRedactions.stream()
.filter(groupRedaction -> groupRedaction.getSoftDeletedTime() == null)
.forEach(groupRedaction -> {
SearchImplementation searchImplementation = new SearchImplementation(groupRedaction.getValue(), false);
bySearchImplementationAsGroup(searchImplementation,
groupRedaction.getTypeId(),
calculateEntityType(groupRedaction.getEntryType()),
groupRedaction.getGroupId(),
groupRedaction.getLegalBasis(),
node);
});
}
public void bySearchImplementationAsGroup(SearchImplementation searchImplementation, String type, EntityType entityType, String groupId, String legalBasis, SemanticNode node) {
Set<Engine> engines = Set.of(Engine.GROUP);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
.ifPresent(entity -> {
if (entity.getEntityType().equals(EntityType.HINT)) {
entity.skip("GRP.0.1", "group hint is skipped by default");
} else {
entity.apply("GRP.0.0", "group entries are applied by default", legalBasis);
}
entity.setGroupId(groupId);
}));
}
private EntityType calculateEntityType(EntryType entryType) {
switch (entryType) {
case FALSE_RECOMMENDATION -> {
return EntityType.FALSE_RECOMMENDATION;
}
case RECOMMENDATION -> {
return EntityType.RECOMMENDATION;
}
case HINT -> {
return EntityType.HINT;
}
case FALSE_POSITIVE -> {
return EntityType.FALSE_POSITIVE;
}
default -> {
return EntityType.ENTITY;
}
}
}
}

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/AreaGroupRedactionService.java Normal file
View File

@ -0,0 +1,28 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.ArrayList;
import java.util.List;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class AreaGroupRedactionService {
public List<PrecursorEntity> createAreaGroupRedactions(List<AreaGroupRedaction> groupRedactions) {
List<PrecursorEntity> precursorEntities = new ArrayList<>();
if (groupRedactions != null) {
groupRedactions.forEach(groupRedaction -> precursorEntities.addAll(PrecursorEntity.fromAreaGroupRedaction(groupRedaction)));
}
return precursorEntities;
}
}

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -11,7 +11,6 @@ import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@ -23,11 +22,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionType;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.utils.UnprocessedUtils;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
@ -46,20 +48,30 @@ public class SectionFinderService {
Document document,
AnalyzeRequest analyzeRequest,
ImportedRedactions importedRedactions,
Set<String> relevantManuallyModifiedAnnotationIds) {
Set<String> relevantManuallyModifiedAnnotationIds,
List<GroupRedaction> groupRedactions) {
long start = System.currentTimeMillis();
Set<Integer> sectionsToReanalyse = new HashSet<>();
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
.stream()
.map(DictionaryIncrementValue::getValue)
.toList(), true);
List<String> unprocessedGroupRedactions = groupRedactions
.stream()
.filter(groupRedaction -> groupRedaction.getGroupRedactionType().equals(GroupRedactionType.TEXT))
.filter(UnprocessedUtils::isUnprocessedGroupedRedaction)
.map(GroupRedaction::getValue)
.toList();
var groupRedactionSearch = new SearchImplementation(unprocessedGroupRedactions, false);
document.streamChildren()
.forEach(mainNode -> {
if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText()) || groupRedactionSearch.atLeastOneMatches(mainNode.getTextBlock()
.getSearchText())) {
sectionsToReanalyse.add(mainNode.getTreeId()
.get(0));
}

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/UnprocessedUtils.java Normal file
View File

@ -0,0 +1,37 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedaction;
import lombok.experimental.UtilityClass;
@UtilityClass
public class UnprocessedUtils {
public static boolean isUnprocessedGroupedRedaction(GroupRedaction groupRedaction) {
if (groupRedaction.getGroupChanges().isEmpty() && groupRedaction.getProcessedDate() == null) {
return true;
}
if (!groupRedaction.getGroupChanges().isEmpty()) {
OffsetDateTime processedDate = groupRedaction.getProcessedDate();
if (processedDate == null) {
return true;
}
OffsetDateTime lastChangeDate = groupRedaction.getGroupChanges().stream()
.map(GroupChange::getChangeDate)
.max(OffsetDateTime::compareTo)
.orElse(null);
return lastChangeDate != null && lastChangeDate.isAfter(processedDate);
}
return false;
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -48,6 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
@ -201,6 +202,9 @@ public abstract class AbstractRedactionIntegrationTest {
@MockBean
protected DictionaryClient dictionaryClient;
@MockBean
protected GroupRedactionClient groupRedactionClient;
@MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -41,8 +43,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.client.GroupRedactionClient;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
@ -52,6 +56,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
@Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class)
@ -229,6 +235,11 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
mockDictionaryCalls(null);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder()
.textGroupRedactions(Collections.emptyList())
.areaGroupRedactions(Collections.emptyList())
.build());
}

158
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/GroupRedactionTest.java Normal file
View File

@ -0,0 +1,158 @@
package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.AreaGroupRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.PositionOnPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.TextGroupRedaction;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(GroupRedactionTest.GroupRedactionTestConfiguration.class)
public class GroupRedactionTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
public static class GroupRedactionTestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
loadDictionaryForTest();
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTemplateDictionaryTypeResponse());
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse());
mockDictionaryCalls(null);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@Test
@SneakyThrows
public void testGroupRedaction() {
String textID = UUID.randomUUID().toString();
String areaID = UUID.randomUUID().toString();
TextGroupRedaction textGroupRedaction = TextGroupRedaction.builder()
.positionOnPage(PositionOnPage.builder().x(423.9f).y(505.5f).width(43.9f).height(15.4f).pageNumber(1).build())
.groupId(textID)
.typeId("CBI_author")
.value("medicine")
.legalBasis("test legal basis")
.entryType(EntryType.ENTITY)
.groupRedactionType(GroupRedactionType.TEXT)
.build();
AreaGroupRedaction areaGroupRedaction = AreaGroupRedaction.builder()
.positionOnPage(PositionOnPage.builder().x(346.27f).y(644.24f).width(219.4f).height(52.24f).pageNumber(2).build())
.groupId(areaID)
.pageRanges(List.of(PageRange.builder().startPage(2).endPage(5).build()))
.typeId("PII")
.value("non-readable content")
.entryType(EntryType.AREA)
.groupRedactionType(GroupRedactionType.AREA)
.section("my section")
.build();
when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder()
.textGroupRedactions(List.of(textGroupRedaction))
.areaGroupRedactions(List.of(areaGroupRedaction))
.build());
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
analyzeService.analyze(request);
System.out.println("Finished analysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
var textGroupRedactions = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getGroupId() != null && e.getGroupId().equals(textID))
.toList();
assertEquals(textGroupRedactions.size(), 6);
var areaGroupRedactions = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getGroupId() != null && e.getGroupId().equals(areaID))
.toList();
assertEquals(areaGroupRedactions.size(), 4);
}
}

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -46,6 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@ -58,6 +61,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@ -106,6 +110,11 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
mockDictionaryCalls(null);
when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder()
.textGroupRedactions(Collections.emptyList())
.areaGroupRedactions(Collections.emptyList())
.build());
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
@ -67,6 +68,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
@ -83,6 +85,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@ -129,6 +132,11 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
mockDictionaryCalls(null);
when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder()
.textGroupRedactions(Collections.emptyList())
.areaGroupRedactions(Collections.emptyList())
.build());
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -42,6 +44,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.group.GroupRedactionInternalResponse;
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
@ -55,6 +58,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import edu.emory.mathcs.backport.java.util.Collections;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@ -105,6 +109,11 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getDossierDictionaryTypeResponse());
mockDictionaryCalls(null);
when(groupRedactionClient.getGroupRedactions(anyString(), anyString(), anyBoolean())).thenReturn(GroupRedactionInternalResponse.builder()
.textGroupRedactions(Collections.emptyList())
.areaGroupRedactions(Collections.emptyList())
.build());
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}