Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-6929' into 'master'

Browse Source 
RED-6929: fix acceptance tests/rules

Closes RED-6929

See merge request redactmanager/redaction-service!35
This commit is contained in:
Kilian Schüttler 2023-07-06 01:39:39 +02:00
commit 390bb7d381
3 changed files with 47 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java
View File

@ -5,6 +5,7 @@ import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfigura
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER; import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
import static java.lang.String.format;
import java.io.IOException; import java.io.IOException;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
@ -47,7 +48,7 @@ public class RedactionMessageReceiver {
// This prevents from endless retries oom errors. // This prevents from endless retries oom errors.
if (message.getMessageProperties().isRedelivered()) { if (message.getMessageProperties().isRedelivered()) {
var errorMessage = String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", var errorMessage = format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
analyzeRequest.getDossierId(), analyzeRequest.getDossierId(),
analyzeRequest.getFileId()); analyzeRequest.getFileId());
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
@ -73,41 +74,55 @@ public class RedactionMessageReceiver {
switch (analyzeRequest.getMessageType()) { switch (analyzeRequest.getMessageType()) {
case REANALYSE: case REANALYSE:
log.info("------------------------------Reanalysis-------------------------------------"); log.info("");
log.info("------------------------------Reanalysis------------------------------------------");
log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.debug(analyzeRequest.getManualRedactions().toString()); log.debug(analyzeRequest.getManualRedactions().toString());
result = analyzeService.reanalyze(analyzeRequest); result = analyzeService.reanalyze(analyzeRequest);
log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); log.info("Successfully reanalyzed dossier {} file {} took: {} s",
log.info("----------------------------------------------------------------"); analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------------------------");
break; break;
case STRUCTURE_ANALYSE: case STRUCTURE_ANALYSE:
log.info("");
log.info("------------------------------Structure Analysis----------------------------------"); log.info("------------------------------Structure Analysis----------------------------------");
log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); log.info("Successfully analyzed structure dossier {} file {} took: {} s",
log.info(""); analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------------------------");
break; break;
case ANALYSE: case ANALYSE:
log.info("------------------------------Full Analysis------------------------------------"); log.info("");
log.info("------------------------------Full Analysis---------------------------------------");
log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.debug(analyzeRequest.getManualRedactions().toString()); log.debug(analyzeRequest.getManualRedactions().toString());
result = analyzeService.analyze(analyzeRequest); result = analyzeService.analyze(analyzeRequest);
log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration()); log.info("Successfully analyzed dossier {} file {} took: {} s",
log.info("----------------------------------------------------------------"); analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------------------------");
break; break;
case SURROUNDING_TEXT: case SURROUNDING_TEXT:
log.info("");
log.info("------------------------------Add surrounding Text--------------------------------"); log.info("------------------------------Add surrounding Text--------------------------------");
log.info("Starting surrounding text search for entries {} ", analyzeRequest.getManualRedactions().getEntriesToAdd());
result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
analyzeRequest.getManualRedactions()); analyzeRequest.getManualRedactions());
log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {} s",
analyzeRequest.getDossierId(), analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
result.getDuration()); format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------"); log.info("----------------------------------------------------------------------------------");
break; break;
default: default:

38
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -92,29 +92,29 @@ public class AnalyzeService {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
FileType.ORIGIN)); FileType.ORIGIN));
log.debug("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Map<Integer, List<ClassifiedImage>> pdfImages = null; Map<Integer, List<ClassifiedImage>> pdfImages = null;
if (redactionServiceSettings.isEnableImageClassification()) { if (redactionServiceSettings.isEnableImageClassification()) {
pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.debug("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} }
classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages); classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages);
log.debug("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} catch (Exception e) { } catch (Exception e) {
throw new RedactionException(e); throw new RedactionException(e);
} }
Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc); Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
log.debug("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document); SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document);
log.debug("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
log.debug("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
return AnalyzeResult.builder() return AnalyzeResult.builder()
.dossierId(analyzeRequest.getDossierId()) .dossierId(analyzeRequest.getDossierId())
@ -131,24 +131,24 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.debug("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
log.debug("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId());
entityRedactionService.addDictionaryEntities(dictionary, document); entityRedactionService.addDictionaryEntities(dictionary, document);
log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities); Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities);
log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());
@ -179,9 +179,9 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.debug("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.debug("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// not yet ready for reanalysis // not yet ready for reanalysis
if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) {
return analyze(analyzeRequest); return analyze(analyzeRequest);
@ -193,7 +193,7 @@ public class AnalyzeService {
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement); Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.debug("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (sectionsToReAnalyse.isEmpty()) { if (sectionsToReAnalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
@ -206,16 +206,16 @@ public class AnalyzeService {
} }
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.debug("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
log.debug("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
log.debug("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
log.debug("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId()); List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -76,7 +76,7 @@ public class DictionaryService {
@Timed("redactmanager_updateDictionary") @Timed("redactmanager_updateDictionary")
public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId); log.debug("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId); long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId); var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId);
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {