Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-8589' into 'master'

Browse Source 
RED-8589 - Add "MANUAL" engine to all annotations that has entries in manualChanges

Closes RED-8589

See merge request redactmanager/redaction-service!282
This commit is contained in:
Corina Olariu 2024-02-23 10:50:05 +01:00
commit fd3fe87d90
2 changed files with 114 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -11,6 +11,7 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@ -18,7 +19,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@ -26,6 +26,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@ -67,13 +68,13 @@ public class EntityLogCreatorService {
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
@ -114,21 +115,24 @@ public class EntityLogCreatorService {
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0)))
.collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
Set<String> newEntityIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))))
.get(0))))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
newEntityLogEntries,
analyzeRequest.getManualRedactions(),
analyzeRequest.getAnalysisNumber());
newEntityLogEntries,
analyzeRequest.getManualRedactions(),
analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
@ -147,8 +151,12 @@ public class EntityLogCreatorService {
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundPrecursorEntries.stream().filter(entity -> !entity.removed()).forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
document.streamAllImages()
.filter(entity -> !entity.removed())
.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundPrecursorEntries.stream()
.filter(entity -> !entity.removed())
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
return entries;
}
@ -192,11 +200,13 @@ public class EntityLogCreatorService {
.positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
.containingNodeId(image.getTreeId())
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.section(image.getManualOverwrite().getSection()
.orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent())
.manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(image))
.entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.engines(getEngines(null, image.getManualOverwrite()))
.build();
}
@ -204,7 +214,8 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
String type = precursorEntity.getManualOverwrite().getType()
.orElse(precursorEntity.getType());
boolean isHint = isHint(precursorEntity.getEntityType());
return EntityLogEntry.builder()
.id(precursorEntity.getId())
@ -214,7 +225,8 @@ public class EntityLogCreatorService {
.type(type)
.state(buildEntryState(precursorEntity))
.entryType(buildEntryType(precursorEntity))
.section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
.section(precursorEntity.getManualOverwrite().getSection()
.orElse(precursorEntity.getSection()))
.containingNodeId(Collections.emptyList())
.closestHeadline("")
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
@ -224,13 +236,12 @@ public class EntityLogCreatorService {
.textBefore("")
.startOffset(-1)
.endOffset(-1)
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.engines(precursorEntity.getEngines())
.positions(precursorEntity.getManualOverwrite().getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
//imported is no longer used, frontend should check engines
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
@ -243,14 +254,20 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(TextEntity entity) {
Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
entity.references()
.stream()
.filter(TextEntity::active)
.forEach(ref -> ref.getPositionsOnPagePerPage()
.forEach(pos -> referenceIds.add(pos.getId())));
boolean isHint = isHint(entity.getEntityType());
return EntityLogEntry.builder()
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.value(entity.getManualOverwrite().getValue()
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.type())
.section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
.section(entity.getManualOverwrite().getSection()
.orElse(entity.getDeepestFullyContainingNode().toString()))
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@ -260,7 +277,7 @@ public class EntityLogCreatorService {
.startOffset(entity.getTextRange().start())
.endOffset(entity.getTextRange().end())
.dossierDictionaryEntry(entity.isDossierDictionaryEntry())
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
.engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
//imported is no longer used, frontend should check engines
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
@ -272,6 +289,17 @@ public class EntityLogCreatorService {
}
private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
engines.add(Engine.MANUAL);
}
return engines;
}
private boolean isHint(EntityType entityType) {
return entityType.equals(EntityType.HINT);
@ -317,7 +345,9 @@ public class EntityLogCreatorService {
private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
return legalBasis.stream()
.map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
.collect(Collectors.toList());
}
}

56
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -38,6 +38,7 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
@ -58,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
@ -318,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.filter(entry -> entry.getValue().equals("Oxford University Press"))
.findFirst()
.get();
assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL));
var asyaLyon = redactionLog.getEntityLogEntry()
.stream()
@ -364,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL));
}
@ -401,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
analyzeService.reanalyze(request);
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow();
assertEquals("Expand to Hint", entityLogEntry.getValue());
assertEquals(1, entityLogEntry.getPositions().size());
assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
assertEquals(ManualRedactionType.RESIZE,
entityLogEntry.getManualChanges()
.get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
}
@Test
@SneakyThrows
public void testAddEngineManualToResizeDictionaryEntry() {
String filePath = "files/new/crafted document.pdf";
AnalyzeRequest request = uploadFileToStorage(filePath);
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
ManualRedactions manualRedactions = new ManualRedactions();
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
var dictionaryEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry())
.findFirst()
.get();
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(dictionaryEntry.getId())
.requestDate(OffsetDateTime.now())
.value("Image")
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
.updateDictionary(true)
.build();
manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
request.setManualRedactions(manualRedactions);
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
.findFirst()
.orElseThrow();
assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,
entityLogEntry.getManualChanges()
.get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
}
}