RED-8589 - Add "MANUAL" engine to all annotations that has entries in...
This commit is contained in:
parent
89227ec850
commit
ac38a966c5
@ -11,6 +11,7 @@ import java.util.stream.Collectors;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||||
@ -26,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
|||||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||||
@ -213,11 +215,13 @@ public class EntityLogCreatorService {
|
|||||||
.positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
|
.positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
|
||||||
.containingNodeId(image.getTreeId())
|
.containingNodeId(image.getTreeId())
|
||||||
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
|
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
|
||||||
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
|
.section(image.getManualOverwrite().getSection()
|
||||||
|
.orElse(image.getParent().toString()))
|
||||||
.imageHasTransparency(image.isTransparent())
|
.imageHasTransparency(image.isTransparent())
|
||||||
.manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
|
.manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
|
||||||
.state(buildEntryState(image))
|
.state(buildEntryState(image))
|
||||||
.entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
.entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||||
|
.engines(getEngines(null, image.getManualOverwrite()))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -245,13 +249,12 @@ public class EntityLogCreatorService {
|
|||||||
.textBefore("")
|
.textBefore("")
|
||||||
.startOffset(-1)
|
.startOffset(-1)
|
||||||
.endOffset(-1)
|
.endOffset(-1)
|
||||||
.positions(precursorEntity.getManualOverwrite()
|
.positions(precursorEntity.getManualOverwrite().getPositions()
|
||||||
.getPositions()
|
|
||||||
.orElse(precursorEntity.getEntityPosition())
|
.orElse(precursorEntity.getEntityPosition())
|
||||||
.stream()
|
.stream()
|
||||||
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||||
.toList())
|
.toList())
|
||||||
.engines(precursorEntity.getEngines())
|
.engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
|
||||||
//imported is no longer used, frontend should check engines
|
//imported is no longer used, frontend should check engines
|
||||||
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
|
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
|
||||||
.imported(false)
|
.imported(false)
|
||||||
@ -271,7 +274,8 @@ public class EntityLogCreatorService {
|
|||||||
.legalBasis(entity.legalBasis())
|
.legalBasis(entity.legalBasis())
|
||||||
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
||||||
.type(entity.type())
|
.type(entity.type())
|
||||||
.section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
|
.section(entity.getManualOverwrite().getSection()
|
||||||
|
.orElse(entity.getDeepestFullyContainingNode().toString()))
|
||||||
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
|
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
|
||||||
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
||||||
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
|
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
|
||||||
@ -281,7 +285,7 @@ public class EntityLogCreatorService {
|
|||||||
.startOffset(entity.getTextRange().start())
|
.startOffset(entity.getTextRange().start())
|
||||||
.endOffset(entity.getTextRange().end())
|
.endOffset(entity.getTextRange().end())
|
||||||
.dossierDictionaryEntry(entity.isDossierDictionaryEntry())
|
.dossierDictionaryEntry(entity.isDossierDictionaryEntry())
|
||||||
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
|
.engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
|
||||||
//imported is no longer used, frontend should check engines
|
//imported is no longer used, frontend should check engines
|
||||||
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
|
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
|
||||||
.imported(false)
|
.imported(false)
|
||||||
@ -293,6 +297,17 @@ public class EntityLogCreatorService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
|
||||||
|
|
||||||
|
Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
|
||||||
|
|
||||||
|
if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
|
||||||
|
engines.add(Engine.MANUAL);
|
||||||
|
}
|
||||||
|
return engines;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private boolean isHint(EntityType entityType) {
|
private boolean isHint(EntityType entityType) {
|
||||||
|
|
||||||
return entityType.equals(EntityType.HINT);
|
return entityType.equals(EntityType.HINT);
|
||||||
|
|||||||
@ -38,6 +38,7 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||||
@ -58,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application;
|
|||||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
|
||||||
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
||||||
@ -318,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
|
|||||||
.filter(entry -> entry.getValue().equals("Oxford University Press"))
|
.filter(entry -> entry.getValue().equals("Oxford University Press"))
|
||||||
.findFirst()
|
.findFirst()
|
||||||
.get();
|
.get();
|
||||||
|
assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL));
|
||||||
|
|
||||||
var asyaLyon = redactionLog.getEntityLogEntry()
|
var asyaLyon = redactionLog.getEntityLogEntry()
|
||||||
.stream()
|
.stream()
|
||||||
@ -364,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
|
assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
|
||||||
assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
|
assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
|
||||||
|
assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -401,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
|
|||||||
analyzeService.reanalyze(request);
|
analyzeService.reanalyze(request);
|
||||||
|
|
||||||
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
|
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
|
||||||
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
|
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entry -> entry.getId().equals(annotationId))
|
||||||
|
.findFirst()
|
||||||
|
.orElseThrow();
|
||||||
assertEquals("Expand to Hint", entityLogEntry.getValue());
|
assertEquals("Expand to Hint", entityLogEntry.getValue());
|
||||||
assertEquals(1, entityLogEntry.getPositions().size());
|
assertEquals(1, entityLogEntry.getPositions().size());
|
||||||
assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
|
assertEquals(ManualRedactionType.RESIZE,
|
||||||
|
entityLogEntry.getManualChanges()
|
||||||
|
.get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
|
||||||
|
assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SneakyThrows
|
||||||
|
public void testAddEngineManualToResizeDictionaryEntry() {
|
||||||
|
|
||||||
|
String filePath = "files/new/crafted document.pdf";
|
||||||
|
AnalyzeRequest request = uploadFileToStorage(filePath);
|
||||||
|
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||||
|
AnalyzeResult result = analyzeService.analyze(request);
|
||||||
|
ManualRedactions manualRedactions = new ManualRedactions();
|
||||||
|
|
||||||
|
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
|
||||||
|
var dictionaryEntry = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry())
|
||||||
|
.findFirst()
|
||||||
|
.get();
|
||||||
|
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
|
||||||
|
.annotationId(dictionaryEntry.getId())
|
||||||
|
.requestDate(OffsetDateTime.now())
|
||||||
|
.value("Image")
|
||||||
|
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
|
||||||
|
.updateDictionary(true)
|
||||||
|
.build();
|
||||||
|
manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
|
||||||
|
request.setManualRedactions(manualRedactions);
|
||||||
|
|
||||||
|
analyzeService.reanalyze(request);
|
||||||
|
|
||||||
|
entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
|
||||||
|
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
|
||||||
|
.findFirst()
|
||||||
|
.orElseThrow();
|
||||||
|
assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,
|
||||||
|
entityLogEntry.getManualChanges()
|
||||||
|
.get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
|
||||||
|
assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user