Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-7384-bp: add useful fields to ManualRedactionEntry

Browse Source
This commit is contained in:
Kilian Schüttler 2024-03-18 13:17:11 +01:00
parent 187f7c95e0
commit f641824270
12 changed files with 527 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.89.3"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.349.4"
val persistenceServiceVersion = "2.349.7"
val springBootStarterVersion = "3.1.5"
configurations {

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java
View File

@ -34,7 +34,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
@ -54,7 +53,6 @@ public class RedactionLogToEntityLogMigrationService {
private static final double MATCH_THRESHOLD = 10;
EntityFindingUtility entityFindingUtility;
EntityEnrichmentService entityEnrichmentService;
DictionaryService dictionaryService;
ManualChangesApplicationService manualChangesApplicationService;
@ -67,8 +65,8 @@ public class RedactionLogToEntityLogMigrationService {
MigratedIds migratedIds = entitiesToMigrate.stream()
.collect(new MigratedIdsCollector());
applyManualChanges(entitiesToMigrate, manualRedactions);
log.info("applying manual changes to migrated entities for file {}", fileId);
applyManualChanges(entitiesToMigrate, manualRedactions);
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -4,6 +4,7 @@ import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreator
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
import java.awt.geom.Rectangle2D;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@ -20,6 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
@ -350,18 +352,8 @@ public final class MigrationEntity {
manualChanges.addAll(manualChangesToApply);
manualChangesToApply.forEach(manualChange -> {
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
// Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
// Instead, we add an add_locally change to the db.
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
.positions(manualResizeRedaction.getPositions())
.annotationId(getNewId())
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
.requestDate(manualResizeRedaction.getRequestDate())
.textAfter(manualResizeRedaction.getTextAfter())
.textBefore(manualResizeRedaction.getTextBefore())
.build();
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
manualResizeRedaction.setAnnotationId(newId);
manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
} else {
migratedEntity.getManualOverwrite().addChange(manualChange);
}
@ -379,26 +371,25 @@ public final class MigrationEntity {
.findFirst()
.orElse(manualChanges.get(0)).getUser();
var requestDate = manualChanges.stream()
.filter(mc -> mc instanceof ManualResizeRedaction)
.findFirst()
.orElse(manualChanges.get(0)).getRequestDate();
OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
return ManualRedactionEntry.builder()
.annotationId(newId)
.fileId(fileId)
.user(user)
.requestDate(requestDate)
.type(redactionLogEntry.getType())
.value(redactionLogEntry.getValue())
.reason(redactionLogEntry.getReason())
.legalBasis(redactionLogEntry.getLegalBasis())
.section(redactionLogEntry.getSection())
.requestDate(requestDate)
.rectangle(false)
.addToDictionary(false)
.addToDossierDictionary(false)
.rectangle(false)
.positions(buildPositions(migratedEntity))
.user(user)
.textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore())
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.build();
}

41
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java
View File

@ -47,7 +47,9 @@ public class EntityFindingUtility {
}
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity,
Map<String, List<TextEntity>> entitiesWithSameValue,
double matchThreshold) {
if (precursorEntity.getValue() == null) {
return Optional.empty();
@ -56,7 +58,7 @@ public class EntityFindingUtility {
List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
if (entityIdentifierValueNotFound(possibleEntities)) {
log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
log.info("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
return Optional.empty();
}
@ -66,18 +68,22 @@ public class EntityFindingUtility {
.min(Comparator.comparingDouble(ClosestEntity::getDistance));
if (optionalClosestEntity.isEmpty()) {
log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
log.info("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
return Optional.empty();
}
ClosestEntity closestEntity = optionalClosestEntity.get();
if (closestEntity.getDistance() > matchThreshold) {
log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
precursorEntity.getValue(),
precursorEntity.getEntityPosition().get(0).pageNumber(),
precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
closestEntity.getDistance(),
matchThreshold);
log.info("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
precursorEntity.getValue(),
precursorEntity.getEntityPosition()
.get(0).pageNumber(),
precursorEntity.getEntityPosition()
.stream()
.map(RectangleWithPage::rectangle2D)
.toList(),
closestEntity.getDistance(),
matchThreshold);
return Optional.empty();
}
@ -93,8 +99,14 @@ public class EntityFindingUtility {
private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage()
.stream()
.map(PositionOnPage::getPage)
.map(Page::getNumber)
.collect(Collectors.toSet());
Set<Integer> originalPageNumbers = originalPositions.stream()
.map(RectangleWithPage::pageNumber)
.collect(Collectors.toSet());
return entityPageNumbers.containsAll(originalPageNumbers);
}
@ -105,15 +117,16 @@ public class EntityFindingUtility {
return Double.MAX_VALUE;
}
return originalPositions.stream()
.mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
.average()
.mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average()
.orElse(Double.MAX_VALUE);
}
private static long countRectangles(TextEntity entity) {
return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
return entity.getPositionsOnPagePerPage()
.stream()
.mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java
View File

@ -9,7 +9,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@ -23,29 +22,21 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class EntityFromPrecursorCreationService {
static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
EntityFindingUtility entityFindingUtility;
EntityCreationService entityCreationService;
DictionaryService dictionaryService;
@Autowired
public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
this.entityFindingUtility = entityFindingUtility;
entityCreationService = new EntityCreationService(entityEnrichmentService);
this.dictionaryService = dictionaryService;
}
public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();

45
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -87,15 +87,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);
@ -122,6 +122,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A.");
}
@Test
public void acceptanceTests() throws IOException {
@ -133,8 +134,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis");
EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow();
var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
.orElseThrow();
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
@ -146,8 +149,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow();
var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
.orElseThrow();
var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.APPLIED, asyaLyon2.getState());
@ -168,13 +173,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
.stream()
.filter(entry -> entry.getType().equals(type))
.filter(entry -> entry.getValue().equals(value))
.filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
.filter(entry -> entry.getContainingNodeId()
.get(0).equals(sectionNumber.get(0)));
}
private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) {
return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
return redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getType().equals(type))
.filter(entry -> entry.getValue().equals(value));
}
@ -201,13 +210,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(EntryState.IGNORED,
findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState());
findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY))
.findFirst()
.get().getState());
}
private static IdRemoval buildIdRemoval(String id) {
return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
}
}

23
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -218,10 +218,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ManualRedactions manualRedactions = ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.annotationId("c6be5277f5ee60dc3d83527798b7fe02")
.fileId(TEST_FILE_ID)
.value("Dr. Alan")
.positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.user("user")
.build()))
.build();
@ -656,6 +658,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.fileId("fileId")
.type("signature")
.requestDate(OffsetDateTime.now())
.user("user")
.build()));
request.setManualRedactions(manualRedactions);
@ -710,6 +713,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.annotationId("308dab9015bfafd911568cffe0a7f7de")
.fileId(TEST_FILE_ID)
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
.user("user")
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
.build());
@ -718,6 +722,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.legalBasis("Something")
.user("user")
.requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
.build());
@ -726,6 +731,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.add(IdRemoval.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.user("user")
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
.build());
@ -945,6 +951,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.textBefore("")
.updateDictionary(false)
.textAfter("")
.user("user")
.build();
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
@ -1032,12 +1039,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.user("user")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.user("user")
.legalBasis("Something")
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
@ -1248,6 +1257,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.user("user")
.build()))
.build());
@ -1288,12 +1298,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.user("user")
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.type("CBI_author")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.user("user")
.build()))
.build());
@ -1454,6 +1466,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.updateDictionary(true)
.annotationId(david.getId())
.fileId(TEST_FILE_ID)
.user("user")
.requestDate(OffsetDateTime.now())
.value("David Ksenia")
.positions(List.of(Rectangle.builder()
@ -1481,7 +1495,14 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
private IdRemoval getIdRemoval(String id) {
return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
return IdRemoval.builder()
.annotationId(id)
.removeFromAllDossiers(false)
.fileId(TEST_FILE_ID)
.user("user")
.removeFromDictionary(false)
.requestDate(OffsetDateTime.now())
.build();
}

131
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -127,15 +127,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);
@ -155,29 +155,40 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
String testEntityValue1 = "Desiree";
String testEntityValue2 = "Melanie";
EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count());
assertEquals(2,
redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.count());
assertEquals(2,
redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue2))
.count());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document)
.findFirst()
.get();
String idToResize = redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
.get()
.getId();
.get().getId();
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder()
.annotationId(idToResize)
.value(expandedEntityKeyword)
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
.addToAllDossiers(false)
.updateDictionary(false)
.requestDate(OffsetDateTime.now())
.build());
manualRedactions.getResizeRedactions()
.add(ManualResizeRedaction.builder()
.annotationId(idToResize)
.fileId(TEST_FILE_ID)
.value(expandedEntityKeyword)
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
.addToAllDossiers(false)
.updateDictionary(false)
.requestDate(OffsetDateTime.now())
.build());
request.setManualRedactions(manualRedactions);
analyzeService.reanalyze(request);
@ -188,21 +199,32 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
fileOutputStream.write(annotateResponse.getDocument());
}
EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get();
EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(expandedEntityKeyword))
.findFirst()
.get();
assertEquals(idToResize, resizedEntry.getId());
assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
assertEquals(1,
redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count());
redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.count());
assertEquals(1,
redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED))
.count());
}
private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
(float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
(float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(),
pageNumber);
(float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
(float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(),
pageNumber);
}
@ -219,10 +241,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.build()));
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.build()));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
@ -232,7 +254,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
AnalyzeRequest request = uploadFileToStorage(pdfFile);
request.setManualRedactions(manualRedactions);
@ -242,11 +264,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.requestDate(OffsetDateTime.now())
.build())));
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.requestDate(OffsetDateTime.now())
.build())));
analyzeService.reanalyze(request);
@ -295,7 +317,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument());
}
long end = System.currentTimeMillis();
var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny();
var optionalEntry = redactionLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findAny();
assertTrue(optionalEntry.isPresent());
assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
System.out.println("duration: " + (end - start));
@ -345,9 +370,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertFalse(redactionLog2.getEntityLogEntry()
.stream()
.filter(entry -> entry.getType().equals("published_information"))
.anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
.stream()
.filter(entry -> entry.getType().equals("published_information"))
.anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry()
.stream()
@ -381,15 +406,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
String annotationId = "testAnnotationId";
manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
.annotationId(annotationId)
.requestDate(OffsetDateTime.now())
.type("manual")
.value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
.positions(List.of(//
new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
.build()));
.annotationId(annotationId)
.requestDate(OffsetDateTime.now())
.type("manual")
.value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
.positions(List.of(//
new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
.build()));
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(annotationId)
.requestDate(OffsetDateTime.now())

123
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java
View File

@ -32,18 +32,33 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualResizeRedactionTest() {
Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
Set<TextEntity> biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
TextEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
TextEntity biggerEntity = biggerEntities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(initialId)
.fileId(TEST_FILE_ID)
.user("user")
.value(biggerEntity.getValue())
.positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
.positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage()
.get(0)))
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.build();
@ -55,8 +70,13 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
assertEquals(biggerEntity.getPages(), entity.getPages());
assertEquals(biggerEntity.getValue(), entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage().get(0).getRectanglePerLine(), entity.getPositionsOnPagePerPage().get(0).getRectanglePerLine());
assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage()
.get(0).getRectanglePerLine(),
entity.getPositionsOnPagePerPage()
.get(0).getRectanglePerLine());
assertTrue(entity.resized());
}
@ -65,12 +85,25 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualForceRedactionTest() {
Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
.annotationId(initialId)
.fileId(TEST_FILE_ID)
.user("user")
.legalBasis("Something")
.requestDate(OffsetDateTime.now())
.build();
doAnalysis(document, List.of(manualForceRedaction));
@ -78,8 +111,12 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertFalse(entity.getIntersectingNodes().isEmpty());
assertEquals(1, entity.getPages().size());
assertEquals("David Ksenia", entity.getValue());
assertEquals("Something", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
assertEquals("Something",
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertFalse(entity.removed());
assertTrue(entity.hasManualChanges());
assertTrue(entity.applied());
@ -90,17 +127,26 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualIDRemovalTest() {
Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).user("user").build();
doAnalysis(document, List.of(idRemoval));
assertEquals("David Ksenia", entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertTrue(entity.ignored());
}
@ -109,13 +155,25 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualIDRemovalButAlsoForceRedactionTest() {
Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
.annotationId(initialId)
.legalBasis("Something")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.user("user")
.build();
doAnalysis(document, List.of(manualForceRedaction));
@ -123,7 +181,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertFalse(entity.getIntersectingNodes().isEmpty());
assertEquals(1, entity.getPages().size());
assertEquals("David Ksenia", entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertFalse(entity.removed());
assertFalse(entity.ignored());
}
@ -131,7 +191,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
if (rects1.stream()
.allMatch(rect1 -> rects2.stream()
.anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
return;
}
// use this for nice formatting of error message
@ -143,15 +205,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
double tolerance = 1e-1;
return Math.abs(r1.getX() - r2.getX()) < tolerance &&//
Math.abs(r1.getY() - r2.getY()) < tolerance &&//
Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
Math.abs(r1.getY() - r2.getY()) < tolerance &&//
Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
}
private static List<Rectangle> toAnnotationRectangles(PositionOnPage positionsOnPage) {
return positionsOnPage.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())).toList();
return positionsOnPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber()))
.toList();
}

66
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java
View File

@ -43,7 +43,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
OffsetDateTime start = OffsetDateTime.now();
String reason = "whatever";
Document document = buildGraphNoImages("files/new/crafted document.pdf");
List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document).peek(e -> e.apply("T.0.0", reason)).toList();
List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document)
.peek(e -> e.apply("T.0.0", reason))
.toList();
assertFalse(entities.isEmpty());
TextEntity entity = entities.get(0);
assertTrue(entity.active());
@ -52,10 +54,11 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.resized());
assertFalse(entity.ignored());
assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
String annotationId = entity.getPositionsOnPagePerPage()
.get(0).getId();
// remove first
IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).user("user").annotationId(annotationId).build();
entity.getManualOverwrite().addChange(removal);
assertTrue(entity.ignored());
assertFalse(entity.applied());
@ -65,6 +68,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
ManualForceRedaction forceRedaction = ManualForceRedaction.builder()
.requestDate(start.plusSeconds(1))
.fileId(TEST_FILE_ID)
.user("user")
.annotationId(annotationId)
.legalBasis("coolio")
.build();
@ -73,10 +77,12 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.ignored());
assertFalse(entity.removed());
assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals("coolio",
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
// remove again
IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build();
IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build();
entity.getManualOverwrite().addChange(removal2);
assertTrue(entity.ignored());
assertFalse(entity.applied());
@ -86,6 +92,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
ManualForceRedaction forceRedaction2 = ManualForceRedaction.builder()
.requestDate(start.plusSeconds(2))
.fileId(TEST_FILE_ID)
.user("user")
.annotationId(annotationId)
.legalBasis("coolio")
.build();
@ -93,7 +100,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertTrue(entity.ignored());
assertFalse(entity.applied());
assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override",
entity.buildReasonWithManualChangeDescriptions());
entity.buildReasonWithManualChangeDescriptions());
String legalBasis = "Yeah";
String section = "Some random section!";
@ -103,6 +110,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
.annotationId(annotationId)
.requestDate(start.plusSeconds(4))
.section(section)
.fileId(TEST_FILE_ID)
.user("peter")
.value(value)
.build();
@ -110,16 +118,32 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertTrue(entity.ignored());
assertFalse(entity.applied());
assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed",
entity.buildReasonWithManualChangeDescriptions());
assertEquals(value, entity.getManualOverwrite().getValue().orElse(entity.getValue()));
assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()));
entity.buildReasonWithManualChangeDescriptions());
assertEquals(value,
entity.getManualOverwrite().getValue()
.orElse(entity.getValue()));
assertEquals(legalBasis,
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals(section,
entity.getManualOverwrite().getSection()
.orElse(entity.getDeepestFullyContainingNode().toString()));
ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder().type("type").requestDate(start.plusSeconds(5)).annotationId(annotationId).build();
ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder()
.type("type")
.requestDate(start.plusSeconds(5))
.fileId(TEST_FILE_ID)
.user("user")
.annotationId(annotationId)
.build();
entity.getManualOverwrite().addChange(imageRecategorizationRequest);
assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
assertTrue(entity.getManualOverwrite().getRecategorized().get());
assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type()));
assertTrue(entity.getManualOverwrite().getRecategorized()
.isPresent());
assertTrue(entity.getManualOverwrite().getRecategorized()
.get());
assertEquals("type",
entity.getManualOverwrite().getType()
.orElse(entity.type()));
}
@ -129,7 +153,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
OffsetDateTime start = OffsetDateTime.now();
String reason = "whatever";
Document document = buildGraphNoImages("files/new/crafted document.pdf");
List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document).peek(e -> e.apply("T.0.0", reason)).toList();
List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document)
.peek(e -> e.apply("T.0.0", reason))
.toList();
assertFalse(entities.isEmpty());
TextEntity entity = entities.get(0);
assertTrue(entity.active());
@ -138,10 +164,11 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.resized());
assertFalse(entity.ignored());
assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
String annotationId = entity.getPositionsOnPagePerPage()
.get(0).getId();
// remove first
IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).user("user").build();
entity.getManualOverwrite().addChange(removal);
assertTrue(entity.ignored());
assertFalse(entity.applied());
@ -152,6 +179,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
.requestDate(start.plusSeconds(1))
.fileId(TEST_FILE_ID)
.annotationId(annotationId)
.user("user")
.legalBasis("coolio")
.build();
entity.getManualOverwrite().addChange(forceRedaction);
@ -159,7 +187,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.ignored());
assertFalse(entity.removed());
assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals("coolio",
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
}

51
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -84,7 +84,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
public void testFoundManualAddRedactionAndRemovedHasStateRemoved() {
DocumentAndEntity context = createFoundManualRedaction();
IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
IdRemoval removal = IdRemoval.builder().annotationId("123").user("user").fileId(TEST_FILE_ID).requestDate(OffsetDateTime.now()).build();
context.entity().getManualOverwrite().addChange(removal);
assertTrue(context.entity().removed());
}
@ -95,7 +95,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() {
DocumentAndEntity context = createNotFoundManualRedaction();
IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
IdRemoval removal = IdRemoval.builder().fileId(TEST_FILE_ID).user("user").annotationId("123").requestDate(OffsetDateTime.now()).build();
context.entity().getManualOverwrite().addChange(removal);
assertTrue(context.entity().removed());
}
@ -108,8 +108,11 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
String value = "To: Syngenta Ltd. Jealott’s Hill";
String type = DICTIONARY_AUTHOR;
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
.annotationId("123")
.type(type)
.value(value)
.user("user")
.fileId(TEST_FILE_ID)
.reason("reason")
.legalBasis("n-a")
.section("n-a")
@ -122,17 +125,20 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
assertTrue(document.getEntities().isEmpty());
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
.entriesToAdd(Set.of(
manualRedactionEntry))
.build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertEquals(1, notFoundManualEntities.size());
assertTrue(document.getEntities().isEmpty());
List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
document,
notFoundManualEntities,
new DictionaryVersion(),
0L).getEntityLogEntry();
document,
notFoundManualEntities,
new DictionaryVersion(),
0L).getEntityLogEntry();
assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue());
@ -146,7 +152,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
Document document = buildGraph("files/new/VV-919901.pdf");
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
.toList();
assertFalse(tempEntities.isEmpty());
var tempEntity = tempEntities.get(0);
List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
@ -158,8 +165,11 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
.type("manual")
.annotationId("123")
.value(tempEntity.getValue())
.reason("reason")
.user("user")
.fileId(TEST_FILE_ID)
.legalBasis("n-a")
.section(tempEntity.getDeepestFullyContainingNode().toString())
.rectangle(false)
@ -172,21 +182,28 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty());
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
.entriesToAdd(Set.of(
manualRedactionEntry))
.build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty());
assertEquals(1, document.getEntities().size());
return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get());
return new DocumentAndEntity(document,
document.getEntities()
.stream()
.findFirst()
.get());
}
public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
(float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(),
pageNumber);
(float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(),
pageNumber);
}

252
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -8,6 +8,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -34,7 +35,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
@ -84,6 +84,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
@SpyBean
RabbitTemplate rabbitTemplate;
@BeforeEach
public void stubClients() {
@ -101,21 +102,22 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@Test
@SneakyThrows
public void testManualSurroundingText() {
@ -125,10 +127,20 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
ManualRedactions manualRedactions = new ManualRedactions();
var aoelId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL");
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
List.of(Rectangle.builder()
.topLeftX(355.53775f)
.topLeftY(266.1895f)
.width(29.32224f)
.height(10.048125f)
.page(1)
.build()),
"AOEL");
var notFoundId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random");
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId,
List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()),
"Random");
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@ -147,30 +159,43 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertFalse(unprocessedManualEntities.isEmpty());
assertEquals(unprocessedManualEntities.size(), 2);
Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream()
.filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId))
.findFirst();
assertTrue(optionalUnprocessedManualEntity.isPresent());
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f);
assertEquals(unprocessedManualEntity.getPositions()
.get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions()
.get(0).y(), 266.49002f);
assertEquals(unprocessedManualEntity.getPositions()
.get(0).w(), 29.322266f);
assertEquals(unprocessedManualEntity.getPositions()
.get(0).h(), 11.017679f);
Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream()
.filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId))
.findFirst();
assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions()
.get(0).getPageNumber(), 1);
assertEquals(unprocessedNotFoundManualEntity.getPositions()
.get(0).getRectangle()[0], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions()
.get(0).getRectangle()[1], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions()
.get(0).getRectangle()[2], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions()
.get(0).getRectangle()[3], 1f);
analyzeService.reanalyze(request);
@ -190,10 +215,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).y(), positions.get(0).getTopLeftY());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).w(), positions.get(0).getWidth());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).h(), positions.get(0).getHeight());
}
@ -205,13 +234,37 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
ManualRedactions manualRedactions = new ManualRedactions();
var aoelId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
List.of(Rectangle.builder()
.topLeftX(384.85536f)
.topLeftY(240.8695f)
.width(13.49088f)
.height(10.048125f)
.page(1)
.build()),
"EL");
var cormsId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS");
ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId,
List.of(Rectangle.builder()
.topLeftX(129.86f)
.topLeftY(505.7295f)
.width(35.9904f)
.height(10.048125f)
.page(1)
.build()),
"CoRMS");
var a9Id = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G");
ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id,
List.of(Rectangle.builder()
.topLeftX(140.1096f)
.topLeftY(291.5095f)
.width(37.84512f)
.height(10.048125f)
.page(1)
.build()),
"A9396G");
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@ -238,35 +291,53 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertFalse(unprocessedManualEntities.isEmpty());
assertEquals(unprocessedManualEntities.size(), 3);
var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst();
var resizedAoel = unprocessedManualEntities.stream()
.filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId))
.findFirst();
assertTrue(resizedAoel.isPresent());
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX());
assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY());
assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth());
assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight());
assertEquals(resizedAoel.get().getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(resizedAoel.get().getPositions()
.get(0).y(), positions.get(0).getTopLeftY());
assertEquals(resizedAoel.get().getPositions()
.get(0).w(), positions.get(0).getWidth());
assertEquals(resizedAoel.get().getPositions()
.get(0).h(), positions.get(0).getHeight());
var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst();
var cormsResized = unprocessedManualEntities.stream()
.filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId))
.findFirst();
assertTrue(cormsResized.isPresent());
assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX());
assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY());
assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth());
assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight());
assertEquals(cormsResized.get().getPositions()
.get(0).x(), positions2.get(0).getTopLeftX());
assertEquals(cormsResized.get().getPositions()
.get(0).y(), positions2.get(0).getTopLeftY());
assertEquals(cormsResized.get().getPositions()
.get(0).w(), positions2.get(0).getWidth());
assertEquals(cormsResized.get().getPositions()
.get(0).h(), positions2.get(0).getHeight());
var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst();
var a9Resized = unprocessedManualEntities.stream()
.filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id))
.findFirst();
assertTrue(a9Resized.isPresent());
assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
assertEquals(a9Resized.get().getTextBefore(), "data for S");
assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX());
assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY());
assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth());
assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight());
assertEquals(a9Resized.get().getPositions()
.get(0).x(), positions3.get(0).getTopLeftX());
assertEquals(a9Resized.get().getPositions()
.get(0).y(), positions3.get(0).getTopLeftY());
assertEquals(a9Resized.get().getPositions()
.get(0).w(), positions3.get(0).getWidth());
assertEquals(a9Resized.get().getPositions()
.get(0).h(), positions3.get(0).getHeight());
}
@ -277,7 +348,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
ManualRedactions manualRedactions = new ManualRedactions();
var aoelId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
List.of(Rectangle.builder()
.topLeftX(384.85536f)
.topLeftY(240.8695f)
.width(13.49088f)
.height(10.048125f)
.page(1)
.build()),
"EL");
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = uploadFileToStorage(pdfFile);
@ -301,10 +380,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).y(), positions.get(0).getTopLeftY());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).w(), positions.get(0).getWidth());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).h(), positions.get(0).getHeight());
}
@ -315,7 +398,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
ManualRedactions manualRedactions = new ManualRedactions();
var aoelId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
List.of(Rectangle.builder()
.topLeftX(384.85536f)
.topLeftY(240.8695f)
.width(13.49088f)
.height(10.048125f)
.page(1)
.build()),
"EL");
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = uploadFileToStorage(pdfFile);
@ -339,10 +430,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth());
assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).y(), positions.get(0).getTopLeftY());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).w(), positions.get(0).getWidth());
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).h(), positions.get(0).getHeight());
}
@ -353,7 +448,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
ManualRedactions manualRedactions = new ManualRedactions();
var aoelId = UUID.randomUUID().toString();
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL");
ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId,
List.of(Rectangle.builder()
.topLeftX(384.85536f)
.topLeftY(240.8695f)
.width(13.49088f)
.height(10.048125f)
.page(1)
.build()),
"EL");
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = uploadFileToStorage(pdfFile);
@ -377,25 +480,32 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
private static ManualResizeRedaction prepareManualSizeRedaction(String id, List<Rectangle> positions, String value) {
ManualResizeRedaction manualResizeRedaction = new ManualResizeRedaction();
manualResizeRedaction.setAnnotationId(id);
manualResizeRedaction.setPositions(positions);
manualResizeRedaction.setUpdateDictionary(false);
manualResizeRedaction.setAddToAllDossiers(false);
manualResizeRedaction.setValue(value);
return manualResizeRedaction;
return ManualResizeRedaction.builder()
.annotationId(id)
.fileId("fileId")
.user("user")
.positions(positions)
.updateDictionary(false)
.addToAllDossiers(false)
.value(value)
.requestDate(OffsetDateTime.now())
.build();
}
private static ManualRedactionEntry prepareManualRedactionEntry(String id, List<Rectangle> positions, String value) {
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(id);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setType("CBI_author");
manualRedactionEntry.setValue(value);
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(positions);
return manualRedactionEntry;
return ManualRedactionEntry.builder()
.annotationId(id)
.fileId("fileId")
.user("user")
.type("CBI_author")
.value(value)
.reason("Manual Redaction")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.positions(positions)
.build();
}
}