Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

DM-285: Entity to component mappings with new ruleset

Browse Source 
* refactoring structure
* added custom syntax validation, such that rule files may not be corrupted
* added component drools preparations
* deprecated redactionLog
* integrated entityLog
* integrated new getRules with ruleFileType
* renamed service.graph package -> service.document
* fixed tests

Signed-off-by: Kilian Schuettler <kilian.schuettler@knecon.com>
This commit is contained in:
Kilian Schuettler 2023-09-08 12:11:49 +02:00
parent e5f1615814
commit ee46e73c99
22 changed files with 308 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java
View File

@ -2,6 +2,16 @@ package com.iqser.red.service.redaction.v1.server.model;
import org.kie.api.runtime.KieContainer;
public record KieWrapper( KieContainer container,long rulesVersion) {
public record KieWrapper(KieContainer container, long rulesVersion) {
public static KieWrapper empty() {
return new KieWrapper(null, -1);
}
public boolean isPresent() {
return container != null && rulesVersion >= 0;
}
}

59
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java
View File

@ -1,28 +1,34 @@
package com.iqser.red.service.redaction.v1.server.queue;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
import static java.lang.String.format;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.exception.DroolsTimeoutException;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.*;
import static java.lang.String.format;
@Slf4j
@Service
@ -99,22 +105,27 @@ public class RedactionMessageReceiver {
log.info("");
result.setMessageType(analyzeRequest.getMessageType());
fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
} catch (Exception e) {
if (e instanceof DroolsTimeoutException && !((DroolsTimeoutException) e).isReported()) {
rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId());
} catch (DroolsTimeoutException droolsTimeoutException) {
if (!droolsTimeoutException.isReported()) {
rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId(), droolsTimeoutException.getRuleFileType());
}
log.warn("Failed to process analyze request: {}", analyzeRequest, e);
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
sendAnalysisFailed(analyzeRequest, priority, droolsTimeoutException);
} catch (Exception e) {
sendAnalysisFailed(analyzeRequest, priority, e);
}
}
private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
log.warn("Failed to process analyze request: {}", analyzeRequest, e);
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
}
@RabbitHandler
@RabbitListener(queues = REDACTION_DQL)
public void receiveAnalyzeRequestDQL(Message in) throws IOException {

77
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -14,6 +14,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@ -84,8 +85,11 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis();
var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to Version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -97,41 +101,40 @@ public class AnalyzeService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var wrapper = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to Version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<FileAttribute> addedFileAttributes = entityDroolsExecutionService.executeRules(wrapper.container(),
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
addedFileAttributes,
notFoundManualRedactionEntries,
document.streamAllImages().toList());
ComponentLog componentLog = new ComponentLog();
if (kieWrapperComponentRules.isPresent()) {
List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
allFileAttributes,
notFoundManualRedactionEntries,
document.streamAllImages().toList());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components);
componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, wrapper);
EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, wrapper);
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
return finalizeAnalysis(analyzeRequest,
startTime,
entityLog,
entityLog, componentLog,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(addedFileAttributes));
false, new HashSet<>(allFileAttributes));
}
@ -178,7 +181,18 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis();
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog previousComponentLog = new ComponentLog();
KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
if (kieWrapperComponentRules.isPresent()) {
log.info("Updated component rules to version {} for file {} in dossier {}",
kieWrapperComponentRules.rulesVersion(),
analyzeRequest.getFileId(),
analyzeRequest.getDossierId());
previousComponentLog = redactionStorageService.getComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded previous component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -199,7 +213,7 @@ public class AnalyzeService {
if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest,
startTime,
previousEntityLog,
previousEntityLog, previousComponentLog,
previousRedactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -210,8 +224,8 @@ public class AnalyzeService {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
var wrapper = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
@ -221,21 +235,32 @@ public class AnalyzeService {
sectionsToReAnalyse.forEach(node -> dictionarySearchService.addDictionaryEntities(dictionary, node));
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(wrapper.container(),
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document,
sectionsToReAnalyse,
dictionary,
analyzeRequest.getFileAttributes(),
analyzeRequest.getManualRedactions(),
nerEntities);
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = new ComponentLog();
if (kieWrapperComponentRules.isPresent()) {
List<Component> components = componentDroolsExecutionService.executeRules(document.getEntities(),
allFileAttributes,
notFoundManualRedactionEntries,
document.streamAllImages().toList());
componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
return finalizeAnalysis(analyzeRequest,
startTime,
entityLog,
entityLog, componentLog,
redactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -292,7 +317,7 @@ public class AnalyzeService {
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
EntityLog entityLog,
EntityLog entityLog, ComponentLog componentLog,
RedactionLog redactionLog,
int numberOfPages,
DictionaryVersion dictionaryVersion,

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
@ -19,7 +20,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@Service
public class ComponentLogCreatorService {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components) {
public ComponentLog buildComponentLog(KieContainer container, int analysisNumber, List<Component> components) {
List<ComponentLogEntry> componentLogEntries = components.stream().map(this::buildComponentLogEntry).toList();
return new ComponentLog(analysisNumber, componentLogEntries, -1, -1, -1);

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java
View File

@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class ComponentDroolsExecutionService {
public List<Component> executeRules(Set<TextEntity> entities, List<FileAttribute> addedFileAttributes, List<ManualEntity> notFoundManualRedactionEntries, List<Image> list) {
public List<Component> executeRules(Set<TextEntity> entities, List<FileAttribute> allFileAttributes, List<ManualEntity> notFoundManualRedactionEntries, List<Image> list) {
return Collections.emptyList();
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java
View File

@ -29,7 +29,7 @@ public class DroolsSyntaxValidationService {
public DroolsSyntaxValidation testRules(Rules rules) {
DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), rules.getRuleFileType());
DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules.getRulesString());
DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
return droolsCompilerSyntaxValidation;
}
@ -89,15 +89,27 @@ public class DroolsSyntaxValidationService {
}
private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(String rules) {
private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(Rules rules) {
var versionId = System.currentTimeMillis();
var testRules = "test-rules";
KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules, versionId, rules);
KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
versionId,
rules.getRulesString(),
toPersistenceRuleFileType(rules.getRuleFileType()));
return buildDroolsCompilerSyntaxValidation(kieBuilder);
}
private com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType toPersistenceRuleFileType(RuleFileType ruleFileType) {
return switch (ruleFileType) {
case ENTITY -> com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType.ENTITY;
case COMPONENT -> com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType.COMPONENT;
};
}
private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);

95
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
View File

@ -7,30 +7,24 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.iqser.red.service.redaction.v1.server.exception.DroolsTimeoutException;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import feign.FeignException;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.ReleaseId;
import org.kie.api.runtime.KieContainer;
import org.kie.api.runtime.KieSession;
import org.kie.api.runtime.rule.QueryResults;
import org.kie.api.runtime.rule.QueryResultsRow;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
@ -46,8 +40,6 @@ public class EntityDroolsExecutionService {
EntityEnrichmentService entityEnrichmentService;
DroolsSyntaxValidationFactory droolsSyntaxValidationFactory;
RedactionServiceSettings settings;
@ -111,7 +103,7 @@ public class EntityDroolsExecutionService {
} catch (ExecutionException e) {
kieSession.dispose();
if(e.getCause() instanceof TimeoutException){
throw new DroolsTimeoutException(e, false);
throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
}
throw new RuntimeException(e);
} catch (InterruptedException e) {
@ -134,73 +126,4 @@ public class EntityDroolsExecutionService {
return fileAttributes;
}
public KieWrapper getLatestKieContainer(String dossierTemplateId) {
long version = -1;
try {
version = rulesClient.getVersion(dossierTemplateId);
} catch (FeignException fe) {
if (fe.status() == HttpStatus.UNPROCESSABLE_ENTITY.value()) {
throw new DroolsTimeoutException(fe.getCause(), true);
}
}
return new KieWrapper(getKieContainer(dossierTemplateId, version), version);
}
private ReleaseId getReleaseId(String dossierTemplate, long version) {
KieServices kieServices = KieServices.Factory.get();
return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version));
}
private KieContainer getKieContainer(String dossierTemplateId, long version) {
KieServices kieServices = KieServices.Factory.get();
try {
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version));
} catch (Exception e) {
registerNewKieContainerVersion(dossierTemplateId, version);
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version));
}
}
private void registerNewKieContainerVersion(String dossierTemplateId, long version) {
var rules = rulesClient.getRules(dossierTemplateId);
if (rules == null || StringUtils.isEmpty(rules.getValue())) {
throw new RuntimeException("Rules cannot be empty.");
}
registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue());
}
private KieBuilder registerNewKieContainerVersion(String dossierTemplateId, long version, String rules) {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version));
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
return kieBuilder.buildAll();
}
public DroolsSyntaxValidation testRules(String rules) {
var versionId = System.currentTimeMillis();
var testRules = "test-rules";
KieBuilder kieBuilder = registerNewKieContainerVersion(testRules, versionId, rules);
return droolsSyntaxValidationFactory.buildDroolsSyntaxValidation(kieBuilder);
}
}

43
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java
View File

@ -10,13 +10,16 @@ import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.ReleaseId;
import org.kie.api.runtime.KieContainer;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
import com.knecon.fforesight.tenantcommons.TenantContext;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
@Service
@ -26,52 +29,62 @@ public class KieContainerCreationService {
private final RulesClient rulesClient;
public KieWrapper getLatestKieContainer(String dossierTemplateId) {
public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) {
long version = rulesClient.getVersion(dossierTemplateId, RuleFileType.ENTITY);
long version = -1;
try {
version = rulesClient.getVersion(dossierTemplateId, ruleFileType);
} catch (FeignException fe) {
if (fe.status() == HttpStatus.UNPROCESSABLE_ENTITY.value()) {
throw new DroolsTimeoutException(fe.getCause(), true, ruleFileType);
}
if (fe.status() == HttpStatus.NOT_FOUND.value()) {
return new KieWrapper(null, -1);
}
}
return new KieWrapper(getKieContainer(dossierTemplateId, version), version);
return new KieWrapper(getKieContainer(dossierTemplateId, version, ruleFileType), version);
}
private ReleaseId getReleaseId(String dossierTemplate, long version) {
private ReleaseId getReleaseId(String dossierTemplate, long version, RuleFileType ruleFileType) {
KieServices kieServices = KieServices.Factory.get();
return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version));
return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate + ":" + ruleFileType.name(), String.format("1.%d", version));
}
public KieContainer getKieContainer(String dossierTemplateId, long version) {
public KieContainer getKieContainer(String dossierTemplateId, long version, RuleFileType ruleFileType) {
KieServices kieServices = KieServices.Factory.get();
try {
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version));
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version, ruleFileType));
} catch (Exception e) {
registerNewKieContainerVersion(dossierTemplateId, version);
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version));
registerNewKieContainerVersion(dossierTemplateId, version, ruleFileType);
return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version, ruleFileType));
}
}
private void registerNewKieContainerVersion(String dossierTemplateId, long version) {
private void registerNewKieContainerVersion(String dossierTemplateId, long version, RuleFileType ruleFileType) {
var rules = rulesClient.getRules(dossierTemplateId, RuleFileType.ENTITY);
var rules = rulesClient.getRules(dossierTemplateId, ruleFileType);
if (rules == null || StringUtils.isEmpty(rules.getValue())) {
throw new RuntimeException("Rules cannot be empty.");
throw new IllegalArgumentException("Entity rules cannot be empty.");
}
registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue());
registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue(), ruleFileType);
}
public KieBuilder registerNewKieContainerVersion(String dossierTemplateId, long version, String rules) {
public KieBuilder registerNewKieContainerVersion(String dossierTemplateId, long version, String rules, RuleFileType ruleFileType) {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version));
kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version, ruleFileType));
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -6,6 +6,7 @@ import java.io.InputStream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -139,6 +140,16 @@ public class RedactionStorageService {
}
public ComponentLog getComponentLog(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.COMPONENT_LOG), ComponentLog.class);
} catch (StorageObjectDoesNotExist e) {
throw new NotFoundException("Component Log is not available.");
}
}
@RequiredArgsConstructor
public enum StorageType {
PARSED_DOCUMENT(".json");

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java
View File

@ -1,4 +1,6 @@
package com.iqser.red.service.redaction.v1.server.exception;
package com.iqser.red.service.redaction.v1.server.utils.exception;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import lombok.Data;
@ -9,16 +11,22 @@ public class DroolsTimeoutException extends RuntimeException {
private boolean reported;
public DroolsTimeoutException(Throwable cause, boolean reported) {
private RuleFileType ruleFileType;
public DroolsTimeoutException(Throwable cause, boolean reported, RuleFileType ruleFileType) {
super(DROOLS_TIMEOUT_MESSAGE, cause);
this.reported = reported;
this.ruleFileType = ruleFileType;
}
public DroolsTimeoutException(boolean reported) {
public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) {
super(DROOLS_TIMEOUT_MESSAGE);
this.reported = reported;
this.ruleFileType = ruleFileType;
}
}

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -29,6 +30,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -146,6 +148,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadTypeForTest();
loadNerForTest();

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertTrue;
@ -37,6 +38,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -73,6 +75,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();
@ -166,7 +169,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis");
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation()).filter(e -> e.getMatchedRule().startsWith("CBI.16"))
RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation())
.filter(e -> e.getMatchedRule().startsWith("CBI.16"))
.findAny()
.orElseThrow();
IdRemoval removal = buildIdRemoval(desireeEtAl.getId());

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -50,8 +51,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@ -62,11 +63,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -105,6 +107,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.util.List;
@ -24,6 +25,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -62,6 +64,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();

119
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -1,35 +1,33 @@
package com.iqser.red.service.redaction.v1.server;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import static java.util.Map.entry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigInteger;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@ -48,26 +46,54 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.*;
import java.math.BigInteger;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import static java.util.Map.entry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ExtendWith(SpringExtension.class)
@ -236,6 +262,7 @@ public class RulesTest {
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.awt.Color;
@ -42,6 +43,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@ -101,6 +103,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
import static java.util.stream.Collectors.toMap;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.io.IOException;
@ -31,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -52,6 +54,8 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();
loadNerForTest();

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.manualchanges;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertTrue;
@ -61,6 +62,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@ -110,6 +112,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
loadDictionaryForTest();
loadTypeForTest();

12
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java
View File

@ -22,6 +22,7 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
public static final String FILE_NAME = "test-file";
@Test
@SneakyThrows
public void testFile() {
@ -30,21 +31,18 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
om.registerModule(new JavaTimeModule());
var file = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".pdf").getInputStream();
redactionStorageService.storeObject("dossierId", "fileId", FileType.ORIGIN, file);
redactionStorageService.storeObject(DOSSIER_ID, FILE_ID, FileType.ORIGIN, file);
try {
var nerData = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".ner.json").getInputStream();
redactionStorageService.storeObject("dossierId", "fileId", FileType.NER_ENTITIES, nerData);
redactionStorageService.storeObject(DOSSIER_ID, FILE_ID, FileType.NER_ENTITIES, nerData);
} catch (Exception e) {
log.warn("No NER File Provided");
redactionServiceSettings.setNerServiceEnabled(false);
}
AnalyzeRequest ar = AnalyzeRequest.builder()
.fileId("fileId")
.dossierId("dossierId")
.analysisNumber(1)
.dossierTemplateId("dossierTemplateId")
AnalyzeRequest ar = AnalyzeRequest.builder().fileId(FILE_ID).dossierId(DOSSIER_ID)
.analysisNumber(1).dossierTemplateId(DOSSIER_TEMPLATE_ID)
.lastProcessed(OffsetDateTime.now())
.excludedPages(Set.of())
.fileAttributes(List.of())

16
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.realdata;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
@ -38,6 +38,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
@ -51,6 +52,7 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -65,6 +67,10 @@ import lombok.SneakyThrows;
@AutoConfigureObservability
public class LiveDataIntegrationTest {
public static final String DOSSIER_ID = "dossierId";
public static final String FILE_ID = "fileId";
public static final String DOSSIER_TEMPLATE_ID = "dossierTemplateId";
protected static String BASE_DIR = "performance/";
protected static String EFSA_SANITISATION_GFL_V1 = "dictionaries/EFSA_sanitisation_GFL_v1/";
@ -132,11 +138,13 @@ public class LiveDataIntegrationTest {
//when(tableServiceResponseAdapter.convertTables(anyString(), anyString())).thenReturn(new HashMap<>());
TenantContext.setTenantId("redaction");
when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
when(dictionaryClient.getVersion(DOSSIER_TEMPLATE_ID)).thenReturn(1L);
when(dictionaryClient.getVersionForDossier(DOSSIER_ID)).thenReturn(1L);
var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "rules.drl").getInputStream());
when(rulesClient.getRules(any(), any())).thenReturn(JSONPrimitive.of(rules));
when(rulesClient.getRules(DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(rules));
when(rulesClient.getRules(DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(rules));
doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
ObjectMapper objectMapper = new ObjectMapper();

75
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionServiceTest.java
View File

@ -1,75 +0,0 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.MockitoAnnotations;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService;
import lombok.SneakyThrows;
class DroolsExecutionServiceTest {
@MockBean
RulesClient rulesClient;
@MockBean
EntityEnrichmentService entityEnrichmentService;
@BeforeEach
public void setupMocks() {
MockitoAnnotations.openMocks(this);
}
@Test
@SneakyThrows
void testRules() {
DroolsExecutionService droolsExecutionService = new DroolsExecutionService(rulesClient, entityEnrichmentService, new DroolsSyntaxValidationFactory(), new RedactionServiceSettings());
var rulesFile = new ClassPathResource("drools/rules.drl");
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsExecutionService.testRules(rulesString);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@Test
@SneakyThrows
void testAllRules() {
DroolsExecutionService droolsExecutionService = new DroolsExecutionService(rulesClient, entityEnrichmentService, new DroolsSyntaxValidationFactory(), new RedactionServiceSettings());
var rulesFile = new ClassPathResource("drools/all_rules.drl");
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
DroolsSyntaxValidation droolsSyntaxValidation = droolsExecutionService.testRules(rulesString);
assertTrue(droolsSyntaxValidation.isCompiled());
}
@Test
@SneakyThrows
void testCorruptedRules() {
DroolsExecutionService droolsExecutionService = new DroolsExecutionService(rulesClient, entityEnrichmentService, new DroolsSyntaxValidationFactory(), new RedactionServiceSettings());
var rulesFile = new ClassPathResource("drools/rules.drl");
String rulesString = new String(rulesFile.getInputStream().readAllBytes());
String corruptedRules = rulesString.replaceAll(";", "");
DroolsSyntaxValidation droolsSyntaxValidation = droolsExecutionService.testRules(corruptedRules);
assertFalse(droolsSyntaxValidation.isCompiled());
}
}

17
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/ExceptionProvider.java Normal file
View File

@ -0,0 +1,17 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.util.Collections;
import feign.FeignException;
import feign.Request;
import lombok.experimental.UtilityClass;
@UtilityClass
public class ExceptionProvider {
public FeignException.NotFound getMockNotFoundFeignException() {
return new FeignException.NotFound("", Request.create(Request.HttpMethod.GET, "", Collections.emptyMap(), null, null, null), null, null);
}
}