Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

DM-285: Entity to component mappings with new ruleset

Browse Source 
* refactoring structure
* added custom syntax validation, such that rule files may not be corrupted
* added component drools preparations
* deprecated redactionLog
* integrated entityLog

Signed-off-by: Kilian Schuettler <kilian.schuettler@knecon.com>
This commit is contained in:
Kilian Schuettler 2023-09-07 19:05:42 +02:00
parent 6521c86e9f
commit 9232973a00
37 changed files with 926 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RuleFileType.java Normal file
View File

@ -0,0 +1,7 @@
package com.iqser.red.service.redaction.v1.model;
public enum RuleFileType {
ENTITY,
COMPONENT
}

15
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rules.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Rules {
RuleFileType ruleFileType;
String rulesString;
}

3
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java
View File

@ -5,10 +5,11 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.service.redaction.v1.model.Rules;
public interface RedactionResource {
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
DroolsSyntaxValidation testRules(@RequestBody String rules);
DroolsSyntaxValidation testRules(@RequestBody Rules rules);
}

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.25.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "8.43.0.Final"
val pdfBoxVersion = "3.0.0-alpha2"
val persistenceServiceVersion = "2.160.0"
val persistenceServiceVersion = "DM-285-2"
configurations {
all {

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -4,6 +4,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.service.redaction.v1.model.Rules;
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService;
import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
@ -20,7 +21,7 @@ public class RedactionController implements RedactionResource {
@Override
public DroolsSyntaxValidation testRules(@RequestBody String rules) {
public DroolsSyntaxValidation testRules(@RequestBody Rules rules) {
try {
return droolsSyntaxValidationService.testRules(rules);

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java
View File

@ -22,7 +22,7 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualEntity implements Entity {
// must be mapped into a TextEntity as is for comments to work correctly
// The id must be mapped into a TextEntity as is for comments to work correctly
String id;
String value;
List<RectangleWithPage> entityPosition;

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java Normal file
View File

@ -0,0 +1,24 @@
package com.iqser.red.service.redaction.v1.server.model.document.entity;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class Component {
String value;
String originalValue;
String transformation;
List<Entity> references;
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -10,7 +10,7 @@ import java.util.Map;
import java.util.PriorityQueue;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -50,7 +50,6 @@ public class TextEntity implements Entity {
@Builder.Default
Set<Engine> engines = new HashSet<>();
// inferred on graph insertion
String value;
String textBefore;
@ -201,4 +200,6 @@ public class TextEntity implements Entity {
return sb.toString();
}
}

187
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -14,6 +14,10 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
@ -21,21 +25,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.service.graph.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.service.graph.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.graph.ManualRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.graph.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
@ -55,9 +62,12 @@ public class AnalyzeService {
DictionaryService dictionaryService;
EntityDroolsExecutionService entityDroolsExecutionService;
ComponentDroolsExecutionService componentDroolsExecutionService;
KieContainerCreationService kieContainerCreationService;
DictionarySearchService dictionarySearchService;
RedactionLogCreatorService redactionLogCreatorService;
EntityLogCreatorService entityLogCreatorService;
ComponentLogCreatorService componentLogCreatorService;
RedactionStorageService redactionStorageService;
RedactionChangeLogService redactionChangeLogService;
LegalBasisClient legalBasisClient;
@ -101,43 +111,63 @@ public class AnalyzeService {
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
addedFileAttributes,
notFoundManualRedactionEntries,
document.streamAllImages().toList());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components);
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> redactionLogEntries = entityLogCreatorService.createRedactionLog(document,
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries,
getComments(analyzeRequest));
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, wrapper);
EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, wrapper);
return finalizeAnalysis(analyzeRequest,
startTime,
entityLog,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(addedFileAttributes));
}
private EntityLog createEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
Dictionary dictionary,
KieWrapper wrapper) {
List<EntityLogEntry> entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
redactionLogEntries,
toSimplifiedSectionText(legalBasis),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionary.getVersion().getDossierTemplateVersion(),
dictionary.getVersion().getDossierVersion(),
wrapper.rulesVersion(),
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog.getRedactionLogEntry(),
entityLog.getEntityLogEntry(),
true);
redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, document.getNumberOfPages(), dictionary.getVersion(), false, new HashSet<>(addedFileAttributes));
entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
return entityLog;
}
private static Map<String, List<Comment>> getComments(AnalyzeRequest analyzeRequest) {
private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
if (analyzeRequest.getManualRedactions() == null) {
return Collections.emptyMap();
}
if (analyzeRequest.getManualRedactions().getComments() == null) {
return Collections.emptyMap();
}
return analyzeRequest.getManualRedactions().getComments();
return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
}
@ -147,13 +177,14 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis();
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// not yet ready for reanalysis
if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) {
if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
}
@ -168,6 +199,7 @@ public class AnalyzeService {
if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest,
startTime,
previousEntityLog,
previousRedactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -198,7 +230,28 @@ public class AnalyzeService {
nerEntities);
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> newRedactionLogEntries = entityLogCreatorService.createRedactionLog(document,
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
return finalizeAnalysis(analyzeRequest,
startTime,
entityLog,
redactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
true,
new HashSet<>(allFileAttributes));
}
@Deprecated(forRemoval = true)
private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
RedactionLog previousRedactionLog,
Set<Integer> sectionsToReanalyseIds) {
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries,
getComments(analyzeRequest));
@ -212,19 +265,34 @@ public class AnalyzeService {
previousRedactionLog.getRedactionLogEntry()
.removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return previousRedactionLog;
}
return finalizeAnalysis(analyzeRequest,
startTime,
previousRedactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
true,
new HashSet<>(allFileAttributes));
private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds) {
List<EntityLogEntry> newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newEntityLogEntries,
false);
previousEntityLog.getEntityLogEntry()
.removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
return previousEntityLog;
}
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
EntityLog entityLog,
RedactionLog redactionLog,
int numberOfPages,
DictionaryVersion dictionaryVersion,
@ -233,7 +301,6 @@ public class AnalyzeService {
redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
@ -316,7 +383,51 @@ public class AnalyzeService {
}
public List<RedactionLogLegalBasis> toSimplifiedSectionText(List<LegalBasis> legalBasis) {
@Deprecated(forRemoval = true)
private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualRedactionEntries,
Dictionary dictionary,
KieWrapper wrapper) {
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries,
getComments(analyzeRequest));
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
redactionLogEntries,
toRedactionLogLegalBasis(legalBasis),
dictionary.getVersion().getDossierTemplateVersion(),
dictionary.getVersion().getDossierVersion(),
wrapper.rulesVersion(),
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog.getRedactionLogEntry(),
true);
redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
return redactionLog;
}
private static Map<String, List<Comment>> getComments(AnalyzeRequest analyzeRequest) {
if (analyzeRequest.getManualRedactions() == null) {
return Collections.emptyMap();
}
if (analyzeRequest.getManualRedactions().getComments() == null) {
return Collections.emptyMap();
}
return analyzeRequest.getManualRedactions().getComments();
}
public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
}

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentCreationService.java
View File

@ -1,5 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service;
public class ComponentCreationService {
}

83
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java Normal file
View File

@ -0,0 +1,83 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@Service
public class ComponentLogCreatorService {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components) {
List<ComponentLogEntry> componentLogEntries = components.stream().map(this::buildComponentLogEntry).toList();
return new ComponentLog(analysisNumber, componentLogEntries, -1, -1, -1);
}
private ComponentLogEntry buildComponentLogEntry(Component component) {
return ComponentLogEntry.builder()
.value(component.getValue())
.transformation(component.getTransformation())
.originalValue(component.getOriginalValue())
.componentEntityReferences(toComponentEntityReferences(component.getReferences()))
.build();
}
private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) {
return references.stream().flatMap(this::toComponentEntityReference).toList();
}
private Stream<ComponentEntityReference> toComponentEntityReference(Entity entity) {
if (entity instanceof TextEntity textEntity) {
textEntity.getPositionsOnPagePerPage()
.stream()
.map(positionOnPage -> ComponentEntityReference.builder()
.page(positionOnPage.getPage().getNumber())
.type(textEntity.getType())
.id(positionOnPage.getId())
.ruleIdentifier(textEntity.getMatchedRule().getRuleIdentifier().toString())
.reason(textEntity.buildReasonWithManualChangeDescriptions())
.build());
} else if (entity instanceof Image image) {
return Stream.of(ComponentEntityReference.builder()
.id(image.getId())
.reason(image.buildReasonWithManualChangeDescriptions())
.type(image.getImageType().toString())
.ruleIdentifier(image.getMatchedRule().getRuleIdentifier().toString())
.page(image.getFirstPage().getNumber())
.build());
} else if (entity instanceof ManualEntity manualEntity) {
return manualEntity.getEntityPosition()
.stream()
.collect(Collectors.groupingBy(RectangleWithPage::pageNumber))
.keySet()
.stream()
.map(pageNumber -> ComponentEntityReference.builder()
.id(manualEntity.getId())
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.type(manualEntity.getType())
.ruleIdentifier(manualEntity.getRuleIdentifier())
.page(pageNumber)
.build());
}
throw new UnsupportedOperationException(String.format("Entity with class %s is not implemented yet", entity.getClass()));
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;

142
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -6,24 +6,23 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -37,19 +36,16 @@ public class EntityLogCreatorService {
private final ManualChangeFactory manualChangeFactory;
public List<RedactionLogEntry> createRedactionLog(Document document,
String dossierTemplateId,
List<ManualEntity> notFoundManualRedactionEntries,
Map<String, List<Comment>> comments) {
public List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
List<RedactionLogEntry> entries = new ArrayList<>();
List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities()
.stream()
.filter(EntityLogCreatorService::isEntityOrRecommendationType)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
return entries;
}
@ -60,10 +56,10 @@ public class EntityLogCreatorService {
}
private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) {
Set<String> processedIds = new HashSet<>();
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
@ -73,11 +69,10 @@ public class EntityLogCreatorService {
}
processedIds.add(positionOnPage.getId());
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId);
redactionLogEntry.setId(positionOnPage.getId());
redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId()));
List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber()))
.toList();
@ -90,53 +85,18 @@ public class EntityLogCreatorService {
}
private List<RedactionLogComment> buildRedactionLogComments(Map<String, List<Comment>> commentsPerId, String id) {
if (!commentsPerId.containsKey(id)) {
return Collections.emptyList();
}
List<Comment> comments = commentsPerId.get(id);
if (comments == null || comments.isEmpty()) {
return Collections.emptyList();
}
return toRedactionLogComments(comments);
}
private List<RedactionLogComment> toRedactionLogComments(List<Comment> comments) {
return comments.stream().map(this::toRedactionLogComment).toList();
}
private RedactionLogComment toRedactionLogComment(Comment comment) {
return new RedactionLogComment(comment.getId(),
comment.getUser(),
comment.getText(),
comment.getAnnotationId(),
comment.getFileId(),
comment.getDate(),
comment.getSoftDeletedTime());
}
private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getType(), dossierTemplateId);
return RedactionLogEntry.builder()
return EntityLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.getType())
.redacted(entity.applied())
.isHint(isHint)
.isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
.sectionNumber(sectionNumber)
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@ -149,25 +109,25 @@ public class EntityLogCreatorService {
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
.reference(referenceIds)
.manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(entity))
.entryType(buildEntryType(entity, isHint))
.build();
}
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(type, dossierTemplateId);
return RedactionLogEntry.builder()
return EntityLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
.type(type)
.redacted(manualEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.state(buildEntryState(manualEntity))
.entryType(buildEntryType(manualEntity, isHint))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.sectionNumber(0)
.matchedRule("ManualRedaction")
@ -185,33 +145,29 @@ public class EntityLogCreatorService {
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
.comments(buildRedactionLogComments(comments, manualEntity.getId()))
.build();
}
public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return RedactionLogEntry.builder()
return EntityLogEntry.builder()
.id(image.getId())
.color(getColor(imageType, dossierTemplateId, image.applied()))
.isImage(true)
.type(imageType)
.redacted(image.applied())
.reason(image.buildReasonWithManualChangeDescriptions())
.legalBasis(image.legalBasis())
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
.isHint(isHint)
.isDictionaryEntry(false)
.isRecommendation(false)
.positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
.sectionNumber(image.getTreeId().get(0))
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent())
.manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
.comments(buildRedactionLogComments(comments, image.getId()))
.state(buildEntryState(image))
.entryType(buildEntryType(image, isHint))
.build();
}
@ -232,11 +188,47 @@ public class EntityLogCreatorService {
}
private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
private EntryState buildEntryState(Entity entity) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
(float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(),
if (entity.applied()) {
return EntryState.APPLIED;
} else if (entity.skipped()) {
return EntryState.SKIPPED;
} else if (entity.ignored()) {
return EntryState.IGNORED;
} else {
return EntryState.REMOVED;
}
}
private EntryType buildEntryType(Entity entity, boolean isHint) {
if (entity instanceof TextEntity textEntity) {
return getEntryType(isHint, textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) {
return getEntryType(isHint, manualEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
}
throw new UnsupportedOperationException("Entity subclass %s not implemented!");
}
private static EntryType getEntryType(boolean isHint, EntityType entityType) {
return switch (entityType) {
case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY;
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
};
}
private Position toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Position(new float[]{(float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight()), (float) rectangle2D.getWidth(), -(float) rectangle2D.getHeight()},
pageNumber);
}

110
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java
View File

@ -5,6 +5,9 @@ import java.util.List;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@ -25,6 +28,7 @@ public class ImportedRedactionService {
private final RedactionStorageService redactionStorageService;
@Deprecated(forRemoval = true)
@Timed("redactmanager_processImportedRedactions")
public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId,
String dossierId,
@ -47,6 +51,29 @@ public class ImportedRedactionService {
}
@Timed("redactmanager_processImportedRedactions")
public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
String dossierId,
String fileId,
List<EntityLogEntry> redactionLogEntries,
boolean addImportedRedactions) {
var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
if (importedRedactions == null) {
return redactionLogEntries;
}
redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
if (addImportedRedactions) {
return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
}
return redactionLogEntries;
}
@Deprecated(forRemoval = true)
private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
List<RedactionLogEntry> redactionLogEntries,
ImportedRedactions importedRedactions) {
@ -70,6 +97,40 @@ public class ImportedRedactionService {
}
private List<EntityLogEntry> addImportedRedactionsEntityLogEntries(String dossierTemplateId, List<EntityLogEntry> entityLogEntries, ImportedRedactions importedRedactions) {
for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
for (var importedRedaction : importedRedactionsValues) {
EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
.id(importedRedaction.getId())
.type(IMPORTED_REDACTION_TYPE)
.imported(true)
.state(EntryState.APPLIED)
.positions(toPositions(importedRedaction.getPositions()))
.color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
.build();
entityLogEntries.add(redactionLogEntry);
}
}
return entityLogEntries;
}
private List<Position> toPositions(List<Rectangle> positions) {
return positions.stream().map(this::toPosition).toList();
}
private Position toPosition(Rectangle rectangle) {
return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage());
}
@Deprecated(forRemoval = true)
private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
@ -91,6 +152,27 @@ public class ImportedRedactionService {
}
private void addIntersections(EntityLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
for (Position rectangle : redactionLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (redactionLogEntry.getImportedRedactionIntersections() == null) {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
}
}
}
}
}
}
boolean valueInRange(float value, float min, float max) {
return round(value) >= round(min) && round(value) <= round(max);
@ -111,6 +193,7 @@ public class ImportedRedactionService {
}
@Deprecated(forRemoval = true)
private Rectangle normalize(Rectangle rectangle) {
Rectangle r = new Rectangle();
@ -138,6 +221,33 @@ public class ImportedRedactionService {
}
private Rectangle normalize(Position position) {
Rectangle r = new Rectangle();
Point p = new Point();
if (position.getRectangle()[2] < 0) {
p.setX(position.getRectangle()[0] + position.getRectangle()[2]);
r.setWidth(Math.abs(position.getRectangle()[2]));
} else {
p.setX(position.getRectangle()[0]);
r.setWidth(position.getRectangle()[2]);
}
if (position.getRectangle()[3] < 0) {
p.setY(position.getRectangle()[1] + position.getRectangle()[3]);
r.setHeight(Math.abs(position.getRectangle()[3]));
} else {
p.setY(position.getRectangle()[1]);
r.setHeight(position.getRectangle()[3]);
}
r.setTopLeft(p);
r.setPage(position.getPageNumber());
return r;
}
private float round(float value) {
double d = Math.pow(10, 0);

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java
View File

@ -12,8 +12,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
@Service
public class ManualChangeFactory {

279
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java Normal file
View File

@ -0,0 +1,279 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class RedactionLogCreatorService {
private final DictionaryService dictionaryService;
private final ManualChangeFactory manualChangeFactory;
public List<RedactionLogEntry> createRedactionLog(Document document,
String dossierTemplateId,
List<ManualEntity> notFoundManualRedactionEntries,
Map<String, List<Comment>> comments) {
List<RedactionLogEntry> entries = new ArrayList<>();
document.getEntities()
.stream()
.filter(RedactionLogCreatorService::isEntityOrRecommendationType)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
return entries;
}
private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
}
private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
Set<String> processedIds = new HashSet<>();
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
// Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
if (processedIds.contains(positionOnPage.getId())) {
continue;
}
processedIds.add(positionOnPage.getId());
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
redactionLogEntry.setId(positionOnPage.getId());
redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId()));
List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber()))
.toList();
redactionLogEntry.setPositions(rectanglesPerLine);
redactionLogEntities.add(redactionLogEntry);
}
return redactionLogEntities;
}
private List<RedactionLogComment> buildRedactionLogComments(Map<String, List<Comment>> commentsPerId, String id) {
if (!commentsPerId.containsKey(id)) {
return Collections.emptyList();
}
List<Comment> comments = commentsPerId.get(id);
if (comments == null || comments.isEmpty()) {
return Collections.emptyList();
}
return toRedactionLogComments(comments);
}
private List<RedactionLogComment> toRedactionLogComments(List<Comment> comments) {
return comments.stream().map(this::toRedactionLogComment).toList();
}
private RedactionLogComment toRedactionLogComment(Comment comment) {
return new RedactionLogComment(comment.getId(),
comment.getUser(),
comment.getText(),
comment.getAnnotationId(),
comment.getFileId(),
comment.getDate(),
comment.getSoftDeletedTime());
}
private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getType(), dossierTemplateId);
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.getType())
.redacted(entity.applied())
.isHint(isHint)
.isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
.sectionNumber(sectionNumber)
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
.isDictionaryEntry(entity.isDictionaryEntry())
.textAfter(entity.getTextAfter())
.textBefore(entity.getTextBefore())
.startOffset(entity.getTextRange().start())
.endOffset(entity.getTextRange().end())
.isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
.engines(getEngines(entity))
.reference(referenceIds)
.manualChanges(mapManualChanges(entity.getManualOverwrite(), isHint))
.build();
}
private Set<Engine> getEngines(TextEntity entity) {
return entity.getEngines() != null ? mapToEngines(entity.getEngines()) : Collections.emptySet();
}
private Set<Engine> mapToEngines(Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines) {
return engines.stream().map(engine -> switch (engine) {
case NER -> Engine.NER;
case RULE -> Engine.RULE;
case DICTIONARY -> Engine.DICTIONARY;
default -> null;
}).filter(Objects::nonNull).collect(Collectors.toSet());
}
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
boolean isHint = isHint(type, dossierTemplateId);
return RedactionLogEntry.builder()
.id(manualEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied()))
.reason(manualEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis())
.value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
.type(type)
.redacted(manualEntity.applied())
.isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
.sectionNumber(0)
.matchedRule("ManualRedaction")
.rectangle(manualEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getEntityPosition()
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.collect(Collectors.toList()))
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
.comments(buildRedactionLogComments(comments, manualEntity.getId()))
.build();
}
private List<ManualChange> mapManualChanges(ManualChangeOverwrite manualEntity, boolean isHint) {
return manualChangeFactory.toManualChangeList(manualEntity.getManualChangeLog(), isHint).stream().map(this::mapManualChange).toList();
}
private ManualChange mapManualChange(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange manualChange) {
ManualChange manualChange1 = new ManualChange();
BeanUtils.copyProperties(manualChange, manualChange1);
return manualChange1;
}
public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return RedactionLogEntry.builder()
.id(image.getId())
.color(getColor(imageType, dossierTemplateId, image.applied()))
.isImage(true)
.type(imageType)
.redacted(image.applied())
.reason(image.buildReasonWithManualChangeDescriptions())
.legalBasis(image.legalBasis())
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
.isHint(isHint)
.isDictionaryEntry(false)
.isRecommendation(false)
.positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
.sectionNumber(image.getTreeId().get(0))
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent())
.manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
.comments(buildRedactionLogComments(comments, image.getId()))
.build();
}
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private boolean isHint(String type, String dossierTemplateId) {
return dictionaryService.isHint(type, dossierTemplateId);
}
private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
(float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(),
pageNumber);
}
}

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java
View File

@ -1,7 +1,17 @@
package com.iqser.red.service.redaction.v1.server.service.drools;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.Component;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@ -13,5 +23,9 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class ComponentDroolsExecutionService {
public List<Component> executeRules(Set<TextEntity> entities, List<FileAttribute> addedFileAttributes, List<ManualEntity> notFoundManualRedactionEntries, List<Image> list) {
return Collections.emptyList();
}
}

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java
View File

@ -9,6 +9,8 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.service.redaction.v1.model.RuleFileType;
import com.iqser.red.service.redaction.v1.model.Rules;
import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
@ -24,20 +26,25 @@ public class DroolsSyntaxValidationService {
@SneakyThrows
public DroolsSyntaxValidation testRules(String rules) {
public DroolsSyntaxValidation testRules(Rules rules) {
DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules);
DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), rules.getRuleFileType());
DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules.getRulesString());
droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
return droolsCompilerSyntaxValidation;
}
private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString) {
private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) {
RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileInputStreamString());
RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileInputStreamString());
case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseCopmonentRuleFileInputStreamString());
};
if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) {
customSyntaxValidation.getDroolsSyntaxErrorMessages()
.add(DroolsSyntaxErrorMessage.builder()

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java
View File

@ -24,6 +24,7 @@ public class KieContainerCreationService {
private final RulesClient rulesClient;
public KieWrapper getLatestKieContainer(String dossierTemplateId) {
long version = rulesClient.getVersion(dossierTemplateId);
@ -38,6 +39,8 @@ public class KieContainerCreationService {
KieServices kieServices = KieServices.Factory.get();
return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version));
}
public KieContainer getKieContainer(String dossierTemplateId, long version) {
KieServices kieServices = KieServices.Factory.get();

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/ComponentCreationService.java Normal file
View File

@ -0,0 +1,5 @@
package com.iqser.red.service.redaction.v1.server.service.graph;
public class ComponentCreationService {
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/EntityCreationService.java
View File

@ -17,7 +17,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.kie.api.runtime.KieSession;
import com.google.common.base.Functions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -6,6 +6,7 @@ import java.io.InputStream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
@ -86,6 +87,18 @@ public class RedactionStorageService {
}
@Timed("redactmanager_getRedactionLog")
public EntityLog getEntityLog(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("RedactionLog not available.");
return null;
}
}
@Timed("redactmanager_getDocumentGraph")
public DocumentData getDocumentData(String dossierId, String fileId) {

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java
View File

@ -26,4 +26,19 @@ public class RuleManagementResources {
}
}
@SneakyThrows
public static InputStream getBaseComponentRuleFileInputStream() {
return new ClassPathResource(Path.of(folderPrefix, "base_component_rules.drl").toString()).getInputStream();
}
@SneakyThrows
public static String getBaseCopmonentRuleFileInputStreamString() {
try (var in = getBaseRuleFileInputStream()) {
return new String(in.readAllBytes());
}
}
}

89
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl Normal file
View File

@ -0,0 +1,89 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.service.graph.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
/**
This file contains all the rules that are in the templates.
The Imports from this file are all that are possible and will be imported for all entity files.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0
rule "LDS.0.0: run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end
// Everything above this comment will be prepended to any entity rule file before execution.

3
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl
View File

@ -46,13 +46,14 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -22,7 +22,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -503,7 +503,7 @@ public class RulesTest {
assertThat(savedManualChange).isPresent();
}
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0]));
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(Engine[]::new));
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java
View File

@ -29,7 +29,7 @@ import com.iqser.red.service.redaction.v1.server.service.graph.DocumentGraphMapp
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.graph.ManualEntityCreationService;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -42,7 +42,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest {
@Autowired
private ManualEntityCreationService redactionLogAdapter;
@Autowired
private EntityLogCreatorService entityLogCreatorService;
private RedactionLogCreatorService redactionLogCreatorService;
@BeforeEach
@ -91,7 +91,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest {
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
List<ManualEntity> notFoundManualRedactionEntries = redactionLogAdapter.toRedactionEntity(originalRedactionLog, document);
var migratedRedactionLogEntries = entityLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap());
var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap());
Map<String, RedactionLogEntry> migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
Map<String, RedactionLogEntry> newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));

20
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java
View File

@ -10,6 +10,8 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.service.redaction.v1.model.RuleFileType;
import com.iqser.red.service.redaction.v1.model.Rules;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.service.graph.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService;
@ -41,7 +43,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString));
droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@ -56,7 +58,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString));
droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@ -71,7 +73,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString));
droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@ -86,7 +88,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString));
droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@ -101,7 +103,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString));
droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@ -116,7 +118,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString));
droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@ -132,7 +134,7 @@ class DroolsSyntaxValidationServiceTest {
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
String corruptedRules = rulesString.replaceAll(";", "");
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(corruptedRules);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, corruptedRules));
assertFalse(droolsSyntaxValidation.isCompiled());
}
@ -145,10 +147,10 @@ class DroolsSyntaxValidationServiceTest {
var rulesFile = new ClassPathResource("drools/rules.drl");
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.graph.nodes.ImageType;",
String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
"");
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(corruptedRules);
DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, corruptedRules));
assertFalse(droolsSyntaxValidation.isCompiled());
}

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java
View File

@ -29,7 +29,7 @@ import com.iqser.red.service.redaction.v1.server.service.graph.EntityEnrichmentS
import com.iqser.red.service.redaction.v1.server.service.graph.ManualEntityCreationService;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
import lombok.SneakyThrows;
@ -42,7 +42,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
private ManualEntityCreationService manualEntityCreationService;
@Autowired
private EntityLogCreatorService entityLogCreatorService;
private RedactionLogCreatorService redactionLogCreatorService;
@MockBean
private DictionaryService dictionaryService;
@ -120,7 +120,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
assertEquals(1, notFoundManualEntities.size());
assertTrue(document.getEntities().isEmpty());
List<RedactionLogEntry> redactionLogEntries = entityLogCreatorService.createRedactionLog(document,
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
TEST_DOSSIER_TEMPLATE_ID,
notFoundManualEntities,
Collections.emptyMap());

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -54,12 +54,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
query "getFileAttributes"

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -25,7 +25,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;