Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'hotfix' into 'master'

Browse Source 
hotfix: filter entities with empty value in redactionLog and entityLog

See merge request redactmanager/redaction-service!215
This commit is contained in:
Dominique Eifländer 2023-11-30 13:25:17 +01:00
commit db6c027cac
4 changed files with 69 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -113,7 +113,8 @@ public class AnalyzeService {
entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
dictionaryIncrement.getDictionaryVersion(),
analyzeRequest.getDossierTemplateId(),
false), document,
false),
document,
previousRedactionLog,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
@ -209,7 +210,11 @@ public class AnalyzeService {
dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion());
return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
return finalizeAnalysis(analyzeRequest,
startTime,
kieWrapperComponentRules,
new EntityLogChanges(entityLog, false),
document,
redactionLog,
document.getNumberOfPages(),
dictionary.getVersion(),
@ -226,7 +231,8 @@ public class AnalyzeService {
Set<Integer> sectionsToReanalyseIds) {
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries);
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
@ -236,12 +242,18 @@ public class AnalyzeService {
previousRedactionLog.getRedactionLogEntry()
.removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return previousRedactionLog;
}
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document,
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
KieWrapper kieWrapperComponentRules,
EntityLogChanges entityLogChanges,
Document document,
RedactionLog redactionLog,
int numberOfPages,
DictionaryVersion dictionaryVersion,
@ -284,7 +296,9 @@ public class AnalyzeService {
}
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules,
Document document,
Set<FileAttribute> addedFileAttributes,
EntityLogChanges entityLogChanges,
DictionaryVersion dictionaryVersion) {
@ -293,7 +307,9 @@ public class AnalyzeService {
return;
}
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLogChanges.getEntityLog(),
document,
addedFileAttributes.stream().toList());
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -377,7 +393,8 @@ public class AnalyzeService {
KieWrapper wrapper) {
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -121,6 +121,7 @@ public class EntityLogCreatorService {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> !entry.getType()
@ -150,6 +151,7 @@ public class EntityLogCreatorService {
Set<String> processedIds = new HashSet<>();
document.getEntities()
.stream()
.filter(entity -> !entity.getValue().isEmpty())
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -47,8 +47,12 @@ public class RedactionLogCreatorService {
List<RedactionLogEntry> entries = new ArrayList<>();
Set<String> processIds = new HashSet<>();
document.getEntities().stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(IEntity::active).forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
document.getEntities()
.stream()
.filter(entity -> !entity.getValue().isEmpty())
.filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(IEntity::active)
.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
return entries;
@ -95,7 +99,8 @@ public class RedactionLogCreatorService {
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
boolean isHint = isHint(entity.getEntityType());
return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@ -185,7 +190,9 @@ public class RedactionLogCreatorService {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return RedactionLogEntry.builder().id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
return RedactionLogEntry.builder()
.id(image.getId())
.color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
.isImage(true)
.value(image.value())
.type(imageType)
@ -235,8 +242,14 @@ public class RedactionLogCreatorService {
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
.textAfter("")
.textBefore("")
.startOffset(-1).endOffset(-1).positions(manualEntity.getManualOverwrite().getPositions().orElse(manualEntity.getEntityPosition())
.stream().map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList())
.startOffset(-1)
.endOffset(-1)
.positions(manualEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.engines(Collections.emptySet())
.reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -3,8 +3,10 @@ package com.iqser.red.service.redaction.v1.server.storage;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
@ -90,7 +92,11 @@ public class RedactionStorageService {
public RedactionLog getRedactionLog(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
RedactionLog.class);
redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
return redactionLog;
} catch (StorageObjectDoesNotExist e) {
log.debug("RedactionLog not available.");
return null;
@ -98,13 +104,16 @@ public class RedactionStorageService {
}
@Timed("redactmanager_getRedactionLog")
public EntityLog getEntityLog(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
entityLog.setEntityLogEntry(entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
return entityLog;
} catch (StorageObjectDoesNotExist e) {
log.debug("RedactionLog not available.");
log.debug("EntityLog not available.");
return null;
}
@ -116,10 +125,18 @@ public class RedactionStorageService {
try {
return DocumentData.builder()
.documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), DocumentStructure.class))
.documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), DocumentTextData[].class))
.documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), DocumentPositionData[].class))
.documentPages(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), DocumentPage[].class))
.documentStructure(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
DocumentStructure.class))
.documentTextData(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
DocumentTextData[].class))
.documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION),
DocumentPositionData[].class))
.documentPages(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
DocumentPage[].class))
.build();
} catch (StorageObjectDoesNotExist e) {
log.debug("DocumentData not available.");