Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7934' into 'master'

Browse Source 
RED-7934: Manual Redactions should be removed instead of ignored, when removed by id

Closes RED-7934

See merge request redactmanager/redaction-service!200
This commit is contained in:
Dominique Eifländer 2023-11-17 15:36:34 +01:00
commit d42758b189
3 changed files with 96 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java
View File

@ -86,6 +86,12 @@ public interface SemanticNode {
}
/**
* Checks if the given page number exists in the list of pages.
*
* @param pageNumber the page number to be checked
* @return true if the page number exists, otherwise false
*/
default boolean onPage(int pageNumber) {
return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java
View File

@ -13,6 +13,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@ -102,7 +103,7 @@ public class RedactionChangeLogService {
Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
.filter(entry -> (entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) && !isLastManualChangeRemove(entry))
.collect(Collectors.toSet());
previouslyExistingEntries.forEach(currentExistingEntries::remove);
@ -113,4 +114,12 @@ public class RedactionChangeLogService {
return addedIds;
}
private static boolean isLastManualChangeRemove(RedactionLogEntry redactionLogEntry){
if(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()){
return false;
}
return redactionLogEntry.getManualChanges().get(redactionLogEntry.getManualChanges().size() -1).getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY;
}
}

114
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java
View File

@ -7,6 +7,7 @@ import static org.wildfly.common.Assert.assertFalse;
import static org.wildfly.common.Assert.assertTrue;
import java.awt.geom.Rectangle2D;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
@ -19,12 +20,14 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
@ -35,7 +38,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCr
import lombok.SneakyThrows;
public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTest {
public class ManualEntityTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
@ -63,39 +66,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
@SneakyThrows
public void manualAddRedactionTest() {
Document document = buildGraph("files/new/VV-919901.pdf");
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
assertFalse(tempEntities.isEmpty());
var tempEntity = tempEntities.get(0);
List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
.stream()
.flatMap(redactionPosition -> redactionPosition.getRectanglePerLine()
.stream()
.map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPosition.getPage().getNumber())))
.toList();
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
.type("manual")
.value(tempEntity.getValue())
.reason("reason")
.legalBasis("n-a")
.section(tempEntity.getDeepestFullyContainingNode().toString())
.rectangle(true)
.positions(positions)
.textAfter("")
.textBefore("")
.build();
tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty());
assertEquals(1, document.getEntities().size());
createFoundManualRedaction();
}
@ -103,6 +74,34 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
@SneakyThrows
public void manualAddRedactionFailingTest() {
createNotFoundManualRedaction();
}
@Test
@SneakyThrows
public void testFoundManualAddRedactionAndRemovedHasStateRemoved() {
DocumentAndEntity context = createFoundManualRedaction();
IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
context.entity().getManualOverwrite().addChange(removal);
assertTrue(context.entity().removed());
}
@Test
@SneakyThrows
public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() {
DocumentAndEntity context = createNotFoundManualRedaction();
IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
context.entity().getManualOverwrite().addChange(removal);
assertTrue(context.entity().removed());
}
private DocumentAndEntity createNotFoundManualRedaction() {
Document document = buildGraph("files/new/VV-919901.pdf");
// This is important due to PDFTron Web Viewer reordering the content, such that this string is selectable.
String value = "To: Syngenta Ltd. Jealott’s Hill";
@ -115,6 +114,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
.section("n-a")
.rectangle(true)
.positions(List.of(new Rectangle(new Point(90, 262), 11, 88, 1), new Rectangle(new Point(90, 247), 11, 131, 1)))
.requestDate(OffsetDateTime.now())
.textAfter("")
.textBefore("")
.build();
@ -136,6 +136,47 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue());
assertEquals(type, redactionLogEntries.get(0).getType());
return new DocumentAndEntity(document, notFoundManualEntities.get(0));
}
private DocumentAndEntity createFoundManualRedaction() {
Document document = buildGraph("files/new/VV-919901.pdf");
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
assertFalse(tempEntities.isEmpty());
var tempEntity = tempEntities.get(0);
List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
.stream()
.flatMap(redactionPosition -> redactionPosition.getRectanglePerLine()
.stream()
.map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPosition.getPage().getNumber())))
.toList();
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
.type("manual")
.value(tempEntity.getValue())
.reason("reason")
.legalBasis("n-a")
.section(tempEntity.getDeepestFullyContainingNode().toString())
.rectangle(true)
.positions(positions)
.requestDate(OffsetDateTime.now())
.textAfter("")
.textBefore("")
.build();
tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty());
assertEquals(1, document.getEntities().size());
return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get());
}
@ -147,4 +188,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
pageNumber);
}
private record DocumentAndEntity(Document document, IEntity entity) {
}
}