Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8372: Include additional info about redactions in redaction annotations

Browse Source
This commit is contained in:
Maverick Studer 2024-02-02 11:11:30 +01:00 committed by Dominique Eifländer
parent e43e9ace68
commit 7be530b445
35 changed files with 756 additions and 807 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.86.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.324.0"
val persistenceServiceVersion = "2.326.0"
val springBootStarterVersion = "3.1.5"
configurations {

54
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java
View File

@ -16,13 +16,12 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
@ -182,46 +181,47 @@ public class RedactionLogToEntityLogMigrationService {
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId)))
.peek(migrationEntity -> {
if (migrationEntity.getManualEntity().getEntityType().equals(EntityType.HINT) &&//
if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&//
!migrationEntity.getRedactionLogEntry().isHint() &&//
!migrationEntity.getRedactionLogEntry().isRedacted()) {
migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
} else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
} else if (lastManualChangeIsRemove(migrationEntity)) {
migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
} else if (migrationEntity.getManualEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
} else if (migrationEntity.getManualEntity().isApplied()) {
migrationEntity.getManualEntity()
.apply(migrationEntity.getManualEntity().getRuleIdentifier(),
migrationEntity.getManualEntity().getReason(),
migrationEntity.getManualEntity().getLegalBasis());
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
migrationEntity.getPrecursorEntity().skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getPrecursorEntity().isApplied()) {
migrationEntity.getPrecursorEntity()
.apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(),
migrationEntity.getPrecursorEntity().getReason(),
migrationEntity.getPrecursorEntity().getLegalBasis());
} else {
migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
migrationEntity.getPrecursorEntity().skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
}
})
.toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList());
entitiesToMigrate.stream().map(MigrationEntity::getPrecursorEntity).toList());
for (MigrationEntity migrationEntity : entitiesToMigrate) {
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(),
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
tempEntitiesByValue,
MATCH_THRESHOLD);
if (optionalTextEntity.isEmpty()) {
migrationEntity.setMigratedEntity(migrationEntity.getManualEntity());
migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
migrationEntity.setNewId(migrationEntity.getManualEntity().getId());
migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
continue;
}
TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange());
TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange());
migrationEntity.setMigratedEntity(entity);
migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
}
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
@ -244,15 +244,15 @@ public class RedactionLogToEntityLogMigrationService {
}
private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node);
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity;
}

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -34,7 +34,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public final class MigrationEntity {
private final ManualEntity manualEntity;
private final PrecursorEntity precursorEntity;
private final RedactionLogEntry redactionLogEntry;
private IEntity migratedEntity;
private String oldId;
@ -47,12 +47,12 @@ public final class MigrationEntity {
}
public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
public static PrecursorEntity createManualEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
EntityType entityType = getEntityType(redactionLogEntry, hint);
return ManualEntity.builder()
return PrecursorEntity.builder()
.id(redactionLogEntry.getId())
.value(redactionLogEntry.getValue())
.entityPosition(rectangleWithPages)
@ -164,7 +164,7 @@ public final class MigrationEntity {
entityLogEntry = createEntityLogEntry(image);
} else if (migratedEntity instanceof TextEntity textEntity) {
entityLogEntry = createEntityLogEntry(textEntity);
} else if (migratedEntity instanceof ManualEntity entity) {
} else if (migratedEntity instanceof PrecursorEntity entity) {
entityLogEntry = createEntityLogEntry(entity);
} else {
throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
@ -261,30 +261,30 @@ public final class MigrationEntity {
}
public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
return EntityLogEntry.builder()
.id(manualEntity.getId())
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())
.type(type)
.state(buildEntryState(manualEntity))
.entryType(buildEntryType(manualEntity))
.state(buildEntryState(precursorEntity))
.entryType(buildEntryType(precursorEntity))
.section(redactionLogEntry.getSection())
.textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore())
.containingNodeId(Collections.emptyList())
.closestHeadline("")
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(manualEntity.isDictionaryEntry())
.dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(precursorEntity.isDictionaryEntry())
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getManualOverwrite()
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
@ -343,11 +343,11 @@ public final class MigrationEntity {
if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) {
if (manualEntity.isRectangle()) {
} else if (entity instanceof PrecursorEntity precursorEntity) {
if (precursorEntity.isRectangle()) {
return EntryType.AREA;
}
return getEntryType(manualEntity.getEntityType());
return getEntryType(precursorEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
}

57
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java
View File

@ -1,11 +1,17 @@
package com.iqser.red.service.redaction.v1.server.model;
import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
import java.util.List;
import java.util.Optional;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.UUID;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
@ -25,7 +31,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
public class ManualEntity implements IEntity {
public class PrecursorEntity implements IEntity {
// The id must be mapped into a TextEntity as is for comments to work correctly
String id;
@ -37,23 +43,26 @@ public class ManualEntity implements IEntity {
String type;
String section;
EntityType entityType;
EntryType entryType;
boolean applied;
boolean isDictionaryEntry;
boolean isDossierDictionaryEntry;
boolean rectangle;
Set<Engine> engines;
@Builder.Default
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
ManualChangeOverwrite manualOverwrite;
public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
var entryType = hint ? EntryType.HINT : EntryType.ENTITY;
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
manualChangeOverwrite.addChange(manualRedactionEntry);
return ManualEntity.builder()
return PrecursorEntity.builder()
.id(manualRedactionEntry.getAnnotationId())
.value(manualRedactionEntry.getValue())
.entityPosition(rectangleWithPages)
@ -63,20 +72,22 @@ public class ManualEntity implements IEntity {
.type(manualRedactionEntry.getType())
.section(manualRedactionEntry.getSection())
.entityType(entityType)
.entryType(entryType)
.applied(true)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
.rectangle(manualRedactionEntry.isRectangle())
.manualOverwrite(manualChangeOverwrite)
.engines(Set.of(Engine.MANUAL))
.build();
}
public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
EntityType entityType = getEntityType(entityLogEntry.getEntryType());
return ManualEntity.builder()
return PrecursorEntity.builder()
.id(entityLogEntry.getId())
.value(entityLogEntry.getValue())
.entityPosition(rectangleWithPages)
@ -86,17 +97,43 @@ public class ManualEntity implements IEntity {
.type(entityLogEntry.getType())
.section(entityLogEntry.getSection())
.entityType(entityType)
.entryType(entityLogEntry.getEntryType())
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
.manualOverwrite(new ManualChangeOverwrite(entityType))
.engines(entityLogEntry.getEngines())
.build();
}
public static ManualEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType()).orElse(EntryType.ENTITY);
EntityType entityType = getEntityType(entryType);
String value = Optional.ofNullable(importedRedaction.getValue()).orElse("");
return PrecursorEntity.builder()
.id(importedRedaction.getId())
.value(value)
.entityPosition(rectangleWithPages)
.reason(Optional.ofNullable(importedRedaction.getReason()).orElse(""))
.legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis()).orElse(""))
.type(Optional.ofNullable(importedRedaction.getType()).orElse(IMPORTED_REDACTION_TYPE))
.entityType(entityType)
.entryType(entryType)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
.manualOverwrite(new ManualChangeOverwrite(entityType))
.engines(Set.of(Engine.IMPORTED))
.build();
}
public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
return ManualEntity.builder()
return PrecursorEntity.builder()
.id(UUID.randomUUID().toString())
.value(manualResizeRedaction.getValue())
.entityPosition(rectangleWithPages)
@ -141,10 +178,4 @@ public class ManualEntity implements IEntity {
}
}
private EntityType getEntityType(boolean isHint) {
return isHint ? EntityType.HINT : EntityType.ENTITY;
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -24,7 +24,8 @@ public final class MatchedRule implements Comparable<MatchedRule> {
public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE);
public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
RuleIdentifier ruleIdentifier;
@Builder.Default

72
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -1,12 +1,13 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
@ -18,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -28,7 +30,7 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@ -37,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
@ -72,9 +75,10 @@ public class AnalyzeService {
RedactionChangeLogService redactionChangeLogService;
LegalBasisClient legalBasisClient;
RedactionServiceSettings redactionServiceSettings;
ImportedRedactionService importedRedactionService;
NotFoundImportedEntitiesService notFoundImportedEntitiesService;
SectionFinderService sectionFinderService;
ManualRedactionEntryService manualRedactionEntryService;
ImportedRedactionEntryService importedRedactionEntryService;
ObservedStorageService observedStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@ -93,6 +97,9 @@ public class AnalyzeService {
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// not yet ready for reanalysis
if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
@ -102,13 +109,11 @@ public class AnalyzeService {
new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId());
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement);
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (sectionsToReAnalyse.isEmpty()) {
// do this only to update the imported redactions with unprocessed manual changes if there are changes
importedRedactionService.processImportedRedactions(previousEntityLog, analyzeRequest);
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
dictionaryIncrement.getDictionaryVersion(),
@ -126,15 +131,18 @@ public class AnalyzeService {
true,
Collections.emptySet());
}
KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries).flatMap(Collection::stream).collect(Collectors.toList());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -142,6 +150,7 @@ public class AnalyzeService {
dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
sectionsToReAnalyse,
@ -151,14 +160,16 @@ public class AnalyzeService {
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualOrImportedEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document,
notFoundManualRedactionEntries,
notFoundManualOrImportedEntries,
previousEntityLog,
sectionsToReanalyseIds,
dictionary.getVersion());
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest,
startTime,
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
@ -187,6 +198,9 @@ public class AnalyzeService {
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -194,13 +208,16 @@ public class AnalyzeService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document,
analyzeRequest.getDossierTemplateId());
var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries).flatMap(Collection::stream).collect(Collectors.toList());
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
dictionary,
@ -209,13 +226,15 @@ public class AnalyzeService {
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualOrImportedEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document,
notFoundManualRedactionEntries,
notFoundManualOrImportedEntries,
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest,
startTime,
kieWrapperComponentRules,
@ -232,24 +251,15 @@ public class AnalyzeService {
@Deprecated(forRemoval = true)
private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundEntries,
RedactionLog previousRedactionLog,
Set<Integer> sectionsToReanalyseIds) {
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries);
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newRedactionLogEntries,
false);
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId(), notFoundEntries);
previousRedactionLog.getRedactionLogEntry()
.removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
.removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE));
previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return previousRedactionLog;
}
@ -348,11 +358,9 @@ public class AnalyzeService {
}
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) {
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement, ImportedRedactions importedRedactions) {
return analyzeRequest.getSectionsToReanalyse().isEmpty() //
? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions);
}
@ -393,7 +401,7 @@ public class AnalyzeService {
@Deprecated(forRemoval = true)
private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundManualRedactionEntries,
Dictionary dictionary,
KieWrapper wrapper) {
@ -411,12 +419,6 @@ public class AnalyzeService {
wrapper.rulesVersion(),
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog.getRedactionLogEntry(),
true);
redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
return redactionLog;
}

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
@ -28,7 +29,8 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes){
public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
}
@ -52,11 +54,12 @@ public class DictionarySearchService {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node))
.peek(entity -> entity.setDictionaryEntry(true))
.peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
.forEach(entity -> entity.addEngine(Engine.DICTIONARY));
.stream()
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> {
entity.setDictionaryEntry(true);
entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
}));
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -60,9 +60,8 @@ public class EntityChangeLogService {
OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
// no need to check imported redactions because they will be added with the merged changes after this step
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && !existingIds.contains(entry.getId()))
.filter(entry -> !existingIds.contains(entry.getId()))
.toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));

77
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -21,7 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@ -45,7 +45,6 @@ public class EntityLogCreatorService {
DictionaryService dictionaryService;
ManualChangeFactory manualChangeFactory;
ImportedRedactionService importedRedactionService;
RedactionServiceSettings redactionServiceSettings;
LegalBasisClient legalBasisClient;
EntityChangeLogService entityChangeLogService;
@ -60,11 +59,11 @@ public class EntityLogCreatorService {
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualEntities,
List<PrecursorEntity> notFoundEntities,
DictionaryVersion dictionaryVersion,
long rulesVersion) {
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities);
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
@ -77,12 +76,8 @@ public class EntityLogCreatorService {
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
// compute changes will be done for the new entries without the imported redactions, since previous imported redactions are present in
// previousExistingEntityLogEntries and have to be ignored in this calculation because the changes will be merged in the next step (processImportedRedactions)
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
// check to see if there are imported redactions and only then recreate them
importedRedactionService.processImportedRedactions(entityLog, analyzeRequest);
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
return entityLog;
}
@ -113,34 +108,33 @@ public class EntityLogCreatorService {
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
List<PrecursorEntity> notFoundEntries,
EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries).stream()
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))
.collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId()
.isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
importedRedactionService.processImportedRedactions(previousEntityLog, analyzeRequest);
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
}
private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
String dossierTemplateId = analyzeRequest.getDossierTemplateId();
List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities()
.stream()
@ -149,7 +143,9 @@ public class EntityLogCreatorService {
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity)));
notFoundPrecursorEntries.stream()
.filter(entity -> !entity.removed())
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
return entries;
}
@ -203,37 +199,40 @@ public class EntityLogCreatorService {
}
private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType());
String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
boolean isHint = isHint(precursorEntity.getEntityType());
return EntityLogEntry.builder()
.id(manualEntity.getId())
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())
.type(type)
.state(buildEntryState(manualEntity))
.entryType(buildEntryType(manualEntity))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.state(buildEntryState(precursorEntity))
.entryType(buildEntryType(precursorEntity))
.section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
.containingNodeId(Collections.emptyList())
.closestHeadline("")
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(manualEntity.isDictionaryEntry())
.dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(precursorEntity.isDictionaryEntry())
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getManualOverwrite()
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.engines(Collections.emptySet())
.engines(precursorEntity.getEngines())
//imported is no longer used, frontend should check engines
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
.reference(Collections.emptySet())
.manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
.manualChanges(manualChangeFactory.toManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), isHint))
.build();
}
@ -259,6 +258,9 @@ public class EntityLogCreatorService {
.endOffset(entity.getTextRange().end())
.dossierDictionaryEntry(entity.isDossierDictionaryEntry())
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
//imported is no longer used, frontend should check engines
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
.reference(referenceIds)
.manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(entity))
@ -291,11 +293,8 @@ public class EntityLogCreatorService {
if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) {
if (manualEntity.isRectangle()) {
return EntryType.AREA;
}
return getEntryType(manualEntity.getEntityType());
} else if (entity instanceof PrecursorEntity precursorEntity) {
return precursorEntity.getEntryType();
}
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
}

195
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionEntriesMergeService.java
View File

@ -1,195 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.google.common.base.Strings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// This is retrieved from persistence service and updated to the needs for imported redactions
@Service
@Slf4j
@RequiredArgsConstructor
public class ImportedRedactionEntriesMergeService {
public List<EntityLogEntry> mergeManualChangesForImportedRedactions(List<EntityLogEntry> importedRedactionsEntityLogEntries, AnalyzeRequest analyzeRequest) {
log.debug("Merging manual changes for imported redactions with analyze number: {}", analyzeRequest.getAnalysisNumber());
Set<String> importedRedactionsIds = importedRedactionsEntityLogEntries.stream().map(e -> e.getId()).collect(Collectors.toSet());
List<BaseAnnotation> allManualChangesForImportedRedactions = allManualChangesForImportedRedactions(analyzeRequest.getManualRedactions(), importedRedactionsIds);
if (allManualChangesForImportedRedactions.isEmpty()) {
log.debug("no changes for imported redactions");
// no changes for the imported redactions
return importedRedactionsEntityLogEntries;
}
final int analysisNumber = analyzeRequest.getAnalysisNumber();
// Sort manual changes by date, so we process them in order of when they were requested
allManualChangesForImportedRedactions = allManualChangesForImportedRedactions.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
allManualChangesForImportedRedactions.forEach(manualChange -> {
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
if (manualChange instanceof IdRemoval idRemoval) {
mergeIdsToRemove(idRemoval, importedRedactionsEntityLogEntries, analysisNumber);
} else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
mergeResizeRedactions(manualResizeRedaction, importedRedactionsEntityLogEntries, analysisNumber);
} else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
mergeLegalBasisChanges(manualLegalBasisChange, importedRedactionsEntityLogEntries, analysisNumber);
} else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
mergeForceRedactions(manualForceRedaction, importedRedactionsEntityLogEntries, analysisNumber);
}
});
return importedRedactionsEntityLogEntries;
}
private List<BaseAnnotation> allManualChangesForImportedRedactions(ManualRedactions manualRedactions, Set<String> importedRedactionsIds) {
log.debug("allManualChangesForImportedRedactions {} manualRedactions resize: {} legalChanges {} removals {} force {}", importedRedactionsIds.size(), manualRedactions.getResizeRedactions().size(), manualRedactions.getLegalBasisChanges().size(),
manualRedactions.getIdsToRemove().size(), manualRedactions.getForceRedactions().size());
List<BaseAnnotation> annotations = Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation)
.filter(baseAnnotation -> importedRedactionsIds.contains(baseAnnotation.getAnnotationId())).toList();
return annotations;
}
public void mergeIdsToRemove(IdRemoval idRemoval, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setState(EntryState.IGNORED);
addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
entityLogEntry.getManualChanges()
.add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(idRemoval.getProcessedDate() == null ? OffsetDateTime.now() : idRemoval.getProcessedDate())
.userId(idRemoval.getUser())
.build());
});
}
public void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualResizeRedaction.getRequestDate());
ManualChange.ManualChangeBuilder manualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.RESIZE)
.requestedDate(manualResizeRedaction.getRequestDate())
.processedDate(manualResizeRedaction.getProcessedDate() == null ? OffsetDateTime.now() : manualResizeRedaction.getProcessedDate())
.userId(manualResizeRedaction.getUser());
entityLogEntry.getManualChanges().add(manualChange.build());
});
}
public void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
Map<String, String> propertyChanges = getPropertyChanges(manualLegalBasisChange);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(manualLegalBasisChange.getProcessedDate() == null ? OffsetDateTime.now() : manualLegalBasisChange.getProcessedDate())
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
});
}
private Map<String, String> getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) {
Map<String, String> propertyChanges = new HashMap<>();
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
}
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) {
propertyChanges.put("value", manualLegalBasisChange.getValue());
}
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection());
}
return propertyChanges;
}
public void mergeForceRedactions(ManualForceRedaction forceRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate() == null ? OffsetDateTime.now() : forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
});
}
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) {
changes.add(Change.builder()
.analysisNumber(analysisNumber)
.dateTime(offsetDateTime)
.type(changeType)
.build());
} else {
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
}
}
private List<Position> convertPositions(List<Rectangle> rectangles) {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
}
}

339
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java
View File

@ -1,339 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class ImportedRedactionService {
public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
private final DictionaryService dictionaryService;
private final RedactionStorageService redactionStorageService;
private final ImportedRedactionEntriesMergeService importedRedactionEntriesMergeService;
@Deprecated(forRemoval = true)
@Timed("redactmanager_processImportedRedactions")
public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId,
String dossierId,
String fileId,
List<RedactionLogEntry> redactionLogEntries,
boolean addImportedRedactions) {
var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
if (importedRedactions == null) {
return redactionLogEntries;
}
redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
if (addImportedRedactions) {
return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
}
return redactionLogEntries;
}
// Always recreate the imported redactions
@Timed("redactmanager_processImportedRedactions")
public void processImportedRedactions(EntityLog entityLog, AnalyzeRequest analyzeRequest) {
// recreate imported redactions with manual changes
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
if (importedRedactions == null || importedRedactions.getImportedRedactions() == null || importedRedactions.getImportedRedactions().isEmpty()) {
log.debug("checkForImportedRedactions - no imported redactions");
return;
}
List<EntityLogEntry> initialImportedRedactionsEntityLogEntries = addImportedRedactionsEntityLogEntries(analyzeRequest.getDossierTemplateId(), importedRedactions);
//merge manual changes for imported redactions
var newEntityLogsForImportedEntities = importedRedactionEntriesMergeService.mergeManualChangesForImportedRedactions(initialImportedRedactionsEntityLogEntries, analyzeRequest);
//remove the old imported redactions
List<EntityLogEntry> previousImportedRedactionsEntries = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
entityLog.getEntityLogEntry().removeAll(previousImportedRedactionsEntries);
// add all changes to entity log
entityLog.getEntityLogEntry().addAll(newEntityLogsForImportedEntities);
// remove all intersections and update with merged imported redactions
calculateIntersections(newEntityLogsForImportedEntities, entityLog, analyzeRequest.getAnalysisNumber());
}
private void calculateIntersections(List<EntityLogEntry> importedRedactionAndMergedEntries, EntityLog entityLog, int analysisNumber) {
if (!importedRedactionAndMergedEntries.isEmpty()) {
// imported redactions present, intersections must be added with merged imported redactions
Map<Integer, List<EntityLogEntry>> importedRedactionsMap = mapImportedRedactionsOnPage(importedRedactionAndMergedEntries);
entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)).forEach(redactionLogEntry -> {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
});
}
}
@Deprecated(forRemoval = true)
private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
List<RedactionLogEntry> redactionLogEntries,
ImportedRedactions importedRedactions) {
for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
for (var importedRedaction : importedRedactionsValues) {
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
.id(importedRedaction.getId())
.type(IMPORTED_REDACTION_TYPE)
.imported(true)
.redacted(true)
.positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList()))
.color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
.build();
redactionLogEntries.add(redactionLogEntry);
}
}
return redactionLogEntries;
}
private List<EntityLogEntry> addImportedRedactionsEntityLogEntries(String dossierTemplateId, ImportedRedactions importedRedactions) {
List<EntityLogEntry> initialImportedRedactionsEntityLogEntries = new ArrayList<>();
var now = OffsetDateTime.now();
for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
for (var importedRedaction : importedRedactionsValues) {
EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
.id(importedRedaction.getId())
.type(IMPORTED_REDACTION_TYPE)
.entryType(EntryType.ENTITY)
.value(Optional.ofNullable(importedRedaction.getValue()).orElse(""))
.imported(true)
.state(EntryState.APPLIED)
.positions(importedRedaction.getPositions())
.color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
.containingNodeId(Collections.emptyList())
.build();
redactionLogEntry.getChanges().add(new Change(1, ChangeType.ADDED, now));
initialImportedRedactionsEntityLogEntries.add(redactionLogEntry);
}
}
return initialImportedRedactionsEntityLogEntries;
}
private List<Position> toPositions(List<Rectangle> positions) {
return positions.stream().map(this::toPosition).toList();
}
private Position toPosition(Rectangle rectangle) {
return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage());
}
@Deprecated(forRemoval = true)
private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
for (Position importedRedactionPosition : importedRedaction.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (redactionLogEntry.getImportedRedactionIntersections() == null) {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
}
}
}
}
}
}
public Map<Integer, List<EntityLogEntry>> mapImportedRedactionsOnPage(List<EntityLogEntry> importedRedactions) {
Map<Integer, List<EntityLogEntry>> importedRedactionsMap = new HashMap<>();
Set<Integer> pageNumbers = importedRedactions.stream().map(EntityLogEntry::getPositions)
.flatMap(Collection::stream)
.map(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position::getPageNumber)
.collect(Collectors.toSet());
pageNumbers.forEach(pageNumber ->
importedRedactionsMap.put(pageNumber, importedRedactions.stream().filter(i -> pageNumber == i.getPositions().get(0).getPageNumber()).collect(Collectors.toList())));
return importedRedactionsMap;
}
public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<EntityLogEntry>> importedRedactionsMap, int analysisNumber) {
for (Position rectangle : entityLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactionsMap.containsKey(rectangle.getPageNumber())) {
var importedRedactionsOnPage = importedRedactionsMap.get(rectangle.getPageNumber());
for (EntityLogEntry importedRedactionLogEntry : importedRedactionsOnPage) {
for (Position importedRedactionPosition : importedRedactionLogEntry.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (entityLogEntry.getImportedRedactionIntersections() == null) {
entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
entityLogEntry.getImportedRedactionIntersections().add(importedRedactionLogEntry.getId());
if(entityLogEntry.getState() != EntryState.REMOVED) {
entityLogEntry.setState(EntryState.REMOVED);
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
}
}
}
}
}
}
}
private void addIntersections(EntityLogEntry entityLogEntry, ImportedRedactions importedRedactions) {
for (Position rectangle : entityLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
for (Position importedRedactionPosition : importedRedaction.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (entityLogEntry.getImportedRedactionIntersections() == null) {
entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
entityLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
}
}
}
}
}
}
boolean valueInRange(float value, float min, float max) {
return round(value) >= round(min) && round(value) <= round(max);
}
boolean rectOverlap(Rectangle a, Rectangle b) {
boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft().getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(),
a.getTopLeft().getX(),
a.getTopLeft().getX() + a.getWidth());
boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft().getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(),
a.getTopLeft().getY(),
a.getTopLeft().getY() + a.getHeight());
return xOverlap && yOverlap;
}
@Deprecated(forRemoval = true)
private Rectangle normalize(Rectangle rectangle) {
Rectangle r = new Rectangle();
Point p = new Point();
if (rectangle.getWidth() < 0) {
p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
r.setWidth(Math.abs(rectangle.getWidth()));
} else {
p.setX(rectangle.getTopLeft().getX());
r.setWidth(rectangle.getWidth());
}
if (rectangle.getHeight() < 0) {
p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight());
r.setHeight(Math.abs(rectangle.getHeight()));
} else {
p.setY(rectangle.getTopLeft().getY());
r.setHeight(rectangle.getHeight());
}
r.setTopLeft(p);
r.setPage(rectangle.getPage());
return r;
}
private Rectangle normalize(Position position) {
Rectangle r = new Rectangle();
Point p = new Point();
if (position.getRectangle()[2] < 0) {
p.setX(position.getRectangle()[0] + position.getRectangle()[2]);
r.setWidth(Math.abs(position.getRectangle()[2]));
} else {
p.setX(position.getRectangle()[0]);
r.setWidth(position.getRectangle()[2]);
}
if (position.getRectangle()[3] < 0) {
p.setY(position.getRectangle()[1] + position.getRectangle()[3]);
r.setHeight(Math.abs(position.getRectangle()[3]));
} else {
p.setY(position.getRectangle()[1]);
r.setHeight(position.getRectangle()[3]);
}
r.setTopLeft(p);
r.setPage(position.getPageNumber());
return r;
}
private float round(float value) {
double d = Math.pow(10, 0);
return (float) (Math.round(value * d) / d);
}
private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, dossierTemplateId);
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -16,7 +16,7 @@ import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@ -79,7 +79,7 @@ public class ManualChangesApplicationService {
.collect(Collectors.toList()));
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
ManualEntity searchEntity = ManualEntity.fromManualResizeRedaction(manualResizeRedaction);
PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction);
// Loop through nodes starting from the deepest fully containing node all the way to the document node
while (node != null) {

102
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java Normal file
View File

@ -0,0 +1,102 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import org.springframework.stereotype.Service;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class NotFoundImportedEntitiesService {
public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
@Timed("redactmanager_processEntityLog")
public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
// recreate imported redactions with manual changes
if (notFoundEntities == null || notFoundEntities.isEmpty()) {
log.debug("checkForImportedRedactions - no not found imported entities");
return;
}
// remove all intersections and update imported entities
calculateIntersectionsForNotFound(notFoundEntities, entityLog, analyzeRequest.getAnalysisNumber());
}
private void calculateIntersectionsForNotFound(List<PrecursorEntity> notFoundEntities, EntityLog entityLog, int analysisNumber) {
if (!notFoundEntities.isEmpty()) {
// imported redactions present, intersections must be added with merged imported redactions
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities);
entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
});
}
}
public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
Set<Integer> pageNumbers = importedEntities.stream()
.map(PrecursorEntity::getEntityPosition)
.flatMap(Collection::stream)
.map(RectangleWithPage::pageNumber)
.collect(Collectors.toSet());
pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber,
importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList())));
return importedRedactionsMap;
}
public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
for (Position rectangle : entityLogEntry.getPositions()) {
if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {
var importedEntitiesOnPage = importedEntitiesMap.get(rectangle.getPageNumber());
for (PrecursorEntity precursorEntity : importedEntitiesOnPage) {
for (RectangleWithPage rectangleWithPage : precursorEntity.getEntityPosition()) {
if (rectangle.toRectangle2D().intersects(rectangleWithPage.rectangle2D())) {
if (entityLogEntry.getImportedRedactionIntersections() == null) {
entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId());
if (entityLogEntry.getState() != EntryState.REMOVED) {
entityLogEntry.setState(EntryState.REMOVED);
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
}
}
}
}
}
}
}
}

42
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -20,7 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
@ -43,7 +43,7 @@ public class RedactionLogCreatorService {
private final ManualChangeFactory manualChangeFactory;
public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<PrecursorEntity> notFoundManualRedactionEntries) {
List<RedactionLogEntry> entries = new ArrayList<>();
Set<String> processIds = new HashSet<>();
@ -219,40 +219,40 @@ public class RedactionLogCreatorService {
}
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
public RedactionLogEntry createRedactionLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType());
String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
boolean isHint = isHint(precursorEntity.getEntityType());
return RedactionLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.value())
.id(precursorEntity.getId())
.color(getColor(type, dossierTemplateId, precursorEntity.applied(), isHint))
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())
.type(type)
.redacted(manualEntity.applied())
.redacted(precursorEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.isRecommendation(precursorEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(precursorEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || precursorEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
.sectionNumber(0)
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(manualEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(precursorEntity.isRectangle())
.isDictionaryEntry(precursorEntity.isDictionaryEntry())
.isDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getManualOverwrite()
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
.manualChanges(mapManualChanges(precursorEntity.getManualOverwrite(), isHint))
.build();
}

98
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java
View File

@ -21,17 +21,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@ -55,7 +52,7 @@ public class UnprocessedChangesService {
final EntityFindingUtility entityFindingUtility;
final RedactionStorageService redactionStorageService;
final EntityEnrichmentService entityEnrichmentService;
final ManualEntityCreationService manualEntityCreationService;
final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
final DictionaryService dictionaryService;
final ManualChangesApplicationService manualChangesApplicationService;
@ -84,18 +81,22 @@ public class UnprocessedChangesService {
allAnnotationIds.addAll(resizeIds);
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
List<ManualEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList()
.stream().map(ManualEntity::fromEntityLogEntry).toList();
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
.toList()
.stream()
.map(PrecursorEntity::fromEntityLogEntry)
.toList();
if (!manualResizeRedactions.isEmpty()) {
processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
}
List<ManualEntity> notFoundManualEntities = new ArrayList<>();
List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
List<PrecursorEntity> notFoundManualEntities = new ArrayList<>();
List<PrecursorEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
if (!manualEntities.isEmpty()) {
notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
}
document.getEntities().forEach(textEntity -> {
@ -127,24 +128,24 @@ public class UnprocessedChangesService {
private void processManualResizeRedactions(Document document,
List<ManualEntity> manualEntities,
List<PrecursorEntity> manualEntities,
List<UnprocessedManualEntity> unprocessedManualEntities,
List<ManualResizeRedaction> manualResizeRedactions) {
Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
for (ManualEntity manualEntity : manualEntities) {
for (PrecursorEntity precursorEntity : manualEntities) {
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD);
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntities, THRESHOLD);
if (optionalTextEntity.isEmpty()) {
unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity));
unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(precursorEntity));
continue;
}
TextEntity correctEntity = createCorrectEntity(manualEntity, optionalTextEntity.get());
TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get());
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId()))
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
.findFirst();
if (optionalManualResizeRedaction.isPresent()) {
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
@ -156,7 +157,6 @@ public class UnprocessedChangesService {
correctEntity.removeFromGraph();
}
}
}
// remove all temp entities from the graph
@ -164,9 +164,9 @@ public class UnprocessedChangesService {
}
private TextEntity createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) {
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
@ -180,40 +180,40 @@ public class UnprocessedChangesService {
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity;
}
return correctEntity;
}
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) {
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
return UnprocessedManualEntity.builder()
.annotationId(manualEntity.getId())
.textAfter("")
.textBefore("")
.section("")
.positions(manualEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.build();
}
return UnprocessedManualEntity.builder()
.annotationId(precursorEntity.getId())
.textAfter("")
.textBefore("")
.section("")
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.build();
}
private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
return manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.toList();
}
return manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.toList();
}
}

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java
View File

@ -19,7 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -47,35 +47,35 @@ public class EntityFindingUtility {
}
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity manualEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
if (manualEntity.getValue() == null) {
if (precursorEntity.getValue() == null) {
return Optional.empty();
}
List<TextEntity> possibleEntities = entitiesWithSameValue.get(manualEntity.getValue().toLowerCase(Locale.ENGLISH));
List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
if (entityIdentifierValueNotFound(possibleEntities)) {
log.warn("Entity could not be created with manualEntity: {}, due to the value {} not being found anywhere.", manualEntity, manualEntity.getValue());
log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
return Optional.empty();
}
Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
.filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition()))
.map(entity -> ClosestEntity.builder().distance(calculateMinDistance(manualEntity.getEntityPosition(), entity)).textEntity(entity).build())
.filter(entity -> pagesMatch(entity, precursorEntity.getEntityPosition()))
.map(entity -> ClosestEntity.builder().distance(calculateMinDistance(precursorEntity.getEntityPosition(), entity)).textEntity(entity).build())
.min(Comparator.comparingDouble(ClosestEntity::getDistance));
if (optionalClosestEntity.isEmpty()) {
log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition());
log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
return Optional.empty();
}
ClosestEntity closestEntity = optionalClosestEntity.get();
if (closestEntity.getDistance() > matchThreshold) {
log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
manualEntity.getValue(),
manualEntity.getEntityPosition().get(0).pageNumber(),
manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
precursorEntity.getValue(),
precursorEntity.getEntityPosition().get(0).pageNumber(),
precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
closestEntity.getDistance(),
matchThreshold);
return Optional.empty();
@ -150,10 +150,10 @@ public class EntityFindingUtility {
}
public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) {
public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) {
Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
Set<String> entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
if (!pageNumbers.stream().allMatch(node::onPage)) {
throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",

64
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java
View File

@ -3,22 +3,21 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
@ -30,7 +29,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualEntityCreationService {
public class EntityFromPrecursorCreationService {
static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
EntityFindingUtility entityFindingUtility;
@ -39,7 +38,7 @@ public class ManualEntityCreationService {
@Autowired
public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
this.entityFindingUtility = entityFindingUtility;
entityCreationService = new EntityCreationService(entityEnrichmentService);
@ -47,10 +46,10 @@ public class ManualEntityCreationService {
}
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd()
List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> !(idRemovals.stream()
.map(BaseAnnotation::getAnnotationId)
@ -62,7 +61,7 @@ public class ManualEntityCreationService {
.get()
.getRequestDate())))
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> {
if (manualEntity.getEntityType().equals(EntityType.HINT)) {
@ -77,34 +76,46 @@ public class ManualEntityCreationService {
}
public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ImportedRedactions importedRedactions, SemanticNode node) {
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities);
List<PrecursorEntity> importedEntities = importedRedactions.getImportedRedactions()
.stream()
.map(PrecursorEntity::fromImportedEntry)
.peek(importedEntity -> importedEntity.apply("IMP.0.0", importedEntity.getReason(), importedEntity.getLegalBasis()))
.toList();
//we could add image handling here as well and create not only text entities
return toTextEntity(importedEntities, node);
}
List<ManualEntity> notFoundManualEntities = new LinkedList<>();
for (ManualEntity manualEntity : manualEntities) {
Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList());
var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
for (PrecursorEntity precursorEntity : findableEntities) {
Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntitiesByValue, MATCH_THRESHOLD);
if (optionalClosestEntity.isEmpty()) {
notFoundManualEntities.add(manualEntity);
notFoundEntities.add(precursorEntity);
continue;
}
createCorrectEntity(manualEntity, optionalClosestEntity.get());
createCorrectEntity(precursorEntity, optionalClosestEntity.get());
}
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
return notFoundManualEntities;
return notFoundEntities;
}
/**
* Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
*
* @param manualEntity The entity identifier for the RedactionEntity.
* @param closestEntity The closest Boundary to the RedactionEntity.
* @param precursorEntity The entity identifier for the RedactionEntity.
* @param closestEntity The closest Boundary to the RedactionEntity.
*/
private void createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
@ -117,10 +128,11 @@ public class ManualEntityCreationService {
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addEngines(precursorEntity.getEngines());
}
}

60
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java Normal file
View File

@ -0,0 +1,60 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class ImportedRedactionEntryService {
private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@Observed(name = "ImportedRedactionEntryService", contextualName = "add-imported-redaction-entries")
public List<PrecursorEntity> addImportedEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, Document document) {
List<PrecursorEntity> notFoundImportedEntries = Collections.emptyList();
if (!importedRedactions.getImportedRedactions().isEmpty()) {
notFoundImportedEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(importedRedactions, document);
log.info("Added Imported entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
if (notFoundImportedEntries.isEmpty()) {
return Collections.emptyList();
}
List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
for (PrecursorEntity notFoundManualRedactionEntry : notFoundImportedEntries) {
manualChanges.stream()
.filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
.forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));
}
return notFoundImportedEntries;
}
private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
}
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java
View File

@ -10,8 +10,8 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor;
@ -22,23 +22,24 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class ManualRedactionEntryService {
private final ManualEntityCreationService manualEntityCreationService;
private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries")
public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
public List<PrecursorEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList();
if (analyzeRequest.getManualRedactions() != null) {
notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
document, dossierTemplateId);
log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
if (notFoundManualRedactionEntries.isEmpty()) {
return Collections.emptyList();
}
List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
for (ManualEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
for (PrecursorEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
manualChanges.stream()
.filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
.forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -13,6 +13,9 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@ -40,7 +43,11 @@ import lombok.extern.slf4j.Slf4j;
public class SectionFinderService {
@Timed("redactmanager_findSectionsToReanalyse")
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) {
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
EntityLog entityLog,
Document document,
AnalyzeRequest analyzeRequest,
ImportedRedactions importedRedactions) {
long start = System.currentTimeMillis();
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@ -64,6 +71,7 @@ public class SectionFinderService {
});
Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
if (!relevantPagesForReanalysis.isEmpty()) {
sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis));
}
@ -102,6 +110,22 @@ public class SectionFinderService {
}
private static Set<Integer> getRelevantPageNumbersForImportedRedactions(ImportedRedactions importedRedactions, Set<String> idsWithUnprocessedManualChanges) {
if (importedRedactions == null || importedRedactions.getImportedRedactions() == null) {
return Collections.emptySet();
}
return importedRedactions.getImportedRedactions()
.stream()
.filter(importedRedaction -> idsWithUnprocessedManualChanges.contains(importedRedaction.getId()))
.map(ImportedRedaction::getPositions)
.flatMap(Collection::stream)
.map(Position::getPageNumber)
.collect(Collectors.toSet());
}
private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
if (manualRedactions == null) {

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.cache.annotation.Cacheable;
@ -11,6 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
@ -76,10 +78,26 @@ public class RedactionStorageService {
@Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
try {
ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactionsPerPage.class);
return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values().stream().flatMap(List::stream).collect(Collectors.toList()));
} catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available.");
return new ImportedRedactions();
}
}
@Timed("redactmanager_getImportedRedactions")
public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactions.class);
ImportedRedactionsPerPage.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available.");
return null;

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java
View File

@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
public class TextEntityTest {
@ -13,7 +13,7 @@ public class TextEntityTest {
@Test
public void testMatchedRule() {
ManualEntity entity = ManualEntity.builder()
PrecursorEntity entity = PrecursorEntity.builder()
.type("PII")
.entityType(EntityType.ENTITY)
.build();
@ -31,7 +31,7 @@ public class TextEntityTest {
@Test
public void testMatchedRuleWithNonsense() {
ManualEntity entity = ManualEntity.builder()
PrecursorEntity entity = PrecursorEntity.builder()
.type("PII")
.entityType(EntityType.ENTITY)
.build();

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -25,7 +25,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@ -35,17 +35,17 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import lombok.SneakyThrows;
public class ManualEntityTest extends BuildDocumentIntegrationTest {
public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@Autowired
private ManualEntityCreationService manualEntityCreationService;
private EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@Autowired
private EntityLogCreatorService entityLogCreatorService;
@ -122,7 +122,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertEquals(1, notFoundManualEntities.size());
@ -162,7 +162,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
.reason("reason")
.legalBasis("n-a")
.section(tempEntity.getDeepestFullyContainingNode().toString())
.rectangle(true)
.rectangle(false)
.positions(positions)
.requestDate(OffsetDateTime.now())
.textAfter("")
@ -172,7 +172,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty());

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -990,6 +990,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -1560,6 +1560,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1391,6 +1391,29 @@ rule "X.7.0: Remove all images"
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -232,6 +232,29 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1150,6 +1150,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -356,6 +356,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -477,6 +477,29 @@ rule "X.7.0: Remove all images"
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -393,6 +393,29 @@ rule "X.7.0: Remove all images"
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

23
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1584,6 +1584,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

22
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1539,6 +1539,28 @@ rule "X.7.0: Remove all images"
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1

1
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt
View File

@ -7,5 +7,6 @@ X.4.0
X.5.0
X.5.1
X.6.*
X.8.*
FA.*.*
LDS.*.*

1
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt
View File

@ -6,5 +6,6 @@ X.4.0
X.5.0
X.5.1
X.7.0
X.8.*
FA.*.*
LDS.*.*