Merge branch 'RED-6929' into 'master'
RED-6929: fix acceptance tests/rules Closes RED-6929 See merge request redactmanager/redaction-service!34
This commit is contained in:
Kilian Schüttler
commit
8b031fa459
@ -1,13 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
|
||||
@ -17,7 +16,6 @@ import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
@ -52,7 +50,12 @@ public class RedactionMessageReceiver {
|
||||
var errorMessage = String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
|
||||
analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId());
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorMessage, priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
new FileErrorInfo(errorMessage,
|
||||
priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE,
|
||||
"redaction-service",
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
|
||||
throw new AmqpRejectAndDontRequeueException(errorMessage);
|
||||
}
|
||||
|
||||
@ -70,21 +73,33 @@ public class RedactionMessageReceiver {
|
||||
switch (analyzeRequest.getMessageType()) {
|
||||
|
||||
case REANALYSE:
|
||||
log.info("------------------------------Reanalysis-------------------------------------");
|
||||
log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug(analyzeRequest.getManualRedactions().toString());
|
||||
result = analyzeService.reanalyze(analyzeRequest);
|
||||
log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
log.info("----------------------------------------------------------------");
|
||||
break;
|
||||
|
||||
case STRUCTURE_ANALYSE:
|
||||
log.info("------------------------------Structure Analysis----------------------------------");
|
||||
log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
log.info("");
|
||||
break;
|
||||
|
||||
case ANALYSE:
|
||||
log.info("------------------------------Full Analysis------------------------------------");
|
||||
log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug(analyzeRequest.getManualRedactions().toString());
|
||||
result = analyzeService.analyze(analyzeRequest);
|
||||
log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
log.info("----------------------------------------------------------------");
|
||||
break;
|
||||
|
||||
case SURROUNDING_TEXT:
|
||||
log.info("------------------------------Add surrounding Text--------------------------------");
|
||||
result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
analyzeRequest.getManualRedactions());
|
||||
@ -92,6 +107,7 @@ public class RedactionMessageReceiver {
|
||||
analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
result.getDuration());
|
||||
log.info("----------------------------------------------------------------");
|
||||
break;
|
||||
|
||||
default:
|
||||
@ -104,7 +120,9 @@ public class RedactionMessageReceiver {
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to process analyze request: {}", analyzeRequest, e);
|
||||
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
|
||||
}
|
||||
}
|
||||
|
||||
@ -119,7 +137,9 @@ public class RedactionMessageReceiver {
|
||||
OffsetDateTime timestamp = in.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
|
||||
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||
log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp);
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp));
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -84,7 +84,6 @@ public class AnalyzeService {
|
||||
@Timed("redactmanager_analyzeDocumentStructure")
|
||||
public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
|
||||
|
||||
log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
ClassificationDocument classifiedDoc;
|
||||
@ -93,29 +92,29 @@ public class AnalyzeService {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
FileType.ORIGIN));
|
||||
log.info("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
Map<Integer, List<ClassifiedImage>> pdfImages = null;
|
||||
if (redactionServiceSettings.isEnableImageClassification()) {
|
||||
pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
log.info("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
}
|
||||
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages);
|
||||
log.info("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
|
||||
log.info("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document);
|
||||
log.info("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document));
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document));
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
|
||||
log.info("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.dossierId(analyzeRequest.getDossierId())
|
||||
@ -130,27 +129,26 @@ public class AnalyzeService {
|
||||
@Timed("redactmanager_analyze")
|
||||
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
|
||||
|
||||
log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
long startTime = System.currentTimeMillis();
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
|
||||
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
|
||||
log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
entityRedactionService.addDictionaryEntities(dictionary, document);
|
||||
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities);
|
||||
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());
|
||||
|
||||
@ -179,12 +177,11 @@ public class AnalyzeService {
|
||||
@SneakyThrows
|
||||
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
|
||||
|
||||
log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
long startTime = System.currentTimeMillis();
|
||||
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
// not yet ready for reanalysis
|
||||
if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) {
|
||||
return analyze(analyzeRequest);
|
||||
@ -196,7 +193,7 @@ public class AnalyzeService {
|
||||
|
||||
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement);
|
||||
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
|
||||
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
if (sectionsToReAnalyse.isEmpty()) {
|
||||
return finalizeAnalysis(analyzeRequest,
|
||||
@ -209,16 +206,16 @@ public class AnalyzeService {
|
||||
}
|
||||
|
||||
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
|
||||
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
|
||||
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
|
||||
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
|
||||
log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
|
||||
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());
|
||||
|
||||
|
||||
@ -77,7 +77,6 @@ public class ManualRedactionSurroundingTextService {
|
||||
.stream()
|
||||
.map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node))
|
||||
.collect(Collectors.toSet());
|
||||
entities.forEach(RedactionEntity::removeFromGraph);
|
||||
RedactionEntity correctEntity = getEntityOnCorrectPosition(entities, toFindPositions);
|
||||
return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());
|
||||
}
|
||||
|
||||
@ -1336,6 +1336,19 @@ rule "MAN.2.0: Apply force redaction"
|
||||
retract($force);
|
||||
end
|
||||
|
||||
rule "MAN.2.1: Apply force redaction to images"
|
||||
no-loop true
|
||||
salience 128
|
||||
when
|
||||
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
|
||||
$imageToForce: Image(id == $id)
|
||||
then
|
||||
$imageToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
|
||||
$imageToForce.setRemoved(false);
|
||||
$imageToForce.setIgnored(false);
|
||||
update($imageToForce);
|
||||
update($imageToForce.getParent());
|
||||
end
|
||||
|
||||
// Rule unit: MAN.3
|
||||
rule "MAN.3.0: Apply image recategorization"
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
<Root level="warn">
|
||||
<AppenderRef ref="CONSOLE"/>
|
||||
</Root>
|
||||
<Logger name="com.iqser" level="info"/>
|
||||
<Logger name="com.iqser" level="debug"/>
|
||||
</Loggers>
|
||||
|
||||
</Configuration>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user