diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java new file mode 100644 index 00000000..baae9ca4 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@AllArgsConstructor +@Builder +@EqualsAndHashCode +public class UnprocessedManualEntity { + + private String annotationId; + private String textBefore; + private String textAfter; + private int startOffset; + private int endOffset; + private String closestHeadline; + private float[] color; + private List containingNodeId; +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/UnprocessedManualEntityResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/UnprocessedManualEntityResource.java new file mode 100644 index 00000000..51b17d9b --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/UnprocessedManualEntityResource.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.resources; + +import java.util.List; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; + +public interface UnprocessedManualEntityResource { + + @PostMapping(value = "/mergeUnprocessedManualEntities", produces = MediaType.APPLICATION_JSON_VALUE) + List mergeUnprocessedManualEntities(@RequestParam(value = "fileId") String fileId, + @RequestParam(value = "dossierId") String dossierId, + @RequestParam(value = "dossierTemplateId") String dosserTemplateId, + @RequestBody ManualRedactions manualRedactions); +} diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 0b75a18e..57cff282 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.74.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.210.0" +val persistenceServiceVersion = "2.223.0" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/UnprocessedManualEntityController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/UnprocessedManualEntityController.java new file mode 100644 index 00000000..14191275 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/UnprocessedManualEntityController.java @@ -0,0 +1,29 @@ +package com.iqser.red.service.redaction.v1.server.controller; + +import java.util.List; + +import org.springframework.web.bind.annotation.RestController; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; +import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource; +import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RestController +@RequiredArgsConstructor +public class UnprocessedManualEntityController implements UnprocessedManualEntityResource { + + private final UnprocessedChangesService unprocessedChangesService; + + + @Override + public List mergeUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) { + + return unprocessedChangesService.addInformationToUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java new file mode 100644 index 00000000..57ece443 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java @@ -0,0 +1,88 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class UnprocessedChangesService { + + private final RedactionStorageService redactionStorageService; + private final DictionaryService dictionaryService; + private final ManualEntityCreationService manualEntityCreationService; + + + public List addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) { + + Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId)); + List unprocessedManualEntities = new ArrayList<>(); + Set annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet()); + + List notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntitiesConverter(manualRedactions), document); + + document.getEntities().forEach(textEntity -> { + unprocessedManualEntities.add(UnprocessedManualEntity.builder() + .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse("")) + .closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) + .color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT))) + .startOffset(textEntity.getTextRange().start()) + .endOffset(textEntity.getTextRange().end()) + .containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId()) + .textBefore(textEntity.getTextBefore()) + .textAfter(textEntity.getTextAfter()) + .build()); + }); + + notFoundManualEntities.forEach(manualEntity -> { + unprocessedManualEntities.add(UnprocessedManualEntity.builder() + .annotationId(manualEntity.getId()) + .color(getColor(manualEntity.getType(), dossierTemplateId, manualEntity.applied(), manualEntity.getEntityType().equals(EntityType.HINT))) + .closestHeadline("") + .startOffset(-1) + .endOffset(-1) + .containingNodeId(Collections.emptyList()) + .textAfter("") + .textBefore("") + .build()); + }); + + return unprocessedManualEntities; + } + + + private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) { + + if (!isApplied && !isHint) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + private List manualEntitiesConverter(ManualRedactions manualRedactions) { + + return manualRedactions.getEntriesToAdd().stream() + .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, + manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only"))).toList(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java index 1cbeb321..887d3001 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java @@ -66,7 +66,7 @@ public class ManualEntityCreationService { } - private List toTextEntity(List manualEntities, SemanticNode node) { + public List toTextEntity(List manualEntities, SemanticNode node) { Set pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet()); Set entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java new file mode 100644 index 00000000..c0e8d2f5 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -0,0 +1,165 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; +import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest; +import com.iqser.red.service.redaction.v1.server.Application; +import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) +public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest { + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @Import({LayoutParsingServiceProcessorConfiguration.class}) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RedactionIntegrationTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(ObjectMapperFactory.create()); + } + + } + + @Autowired + RedactionStorageService redactionStorageService; + + @MockBean + DictionaryService dictionaryService; + + @Autowired + UnprocessedChangesService unprocessedChangesService; + + @Test + public void testManualSurroundingText() { + + String pdfFile = "files/new/S4.pdf"; + when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f}); + when(dictionaryService.getNotRedactedColor(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f}); + + ManualRedactions manualRedactions = new ManualRedactions(); + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString()); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setStatus(AnnotationStatus.APPROVED); + manualRedactionEntry.setType("CBI_author"); + manualRedactionEntry.setValue("rabbits"); + manualRedactionEntry.setReason("Manual Redaction"); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build())); + + ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry(); + manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString()); + manualRedactionEntry2.setFileId("fileId"); + manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED); + manualRedactionEntry2.setType("CBI_author"); + manualRedactionEntry2.setValue("rabbits"); + manualRedactionEntry2.setReason("Manual Redaction"); + manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build())); + + var aoelId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry(); + manualRedactionEntry3.setAnnotationId(aoelId); + manualRedactionEntry3.setFileId("fileId"); + manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED); + manualRedactionEntry3.setType("CBI_author"); + manualRedactionEntry3.setValue("AOEL"); + manualRedactionEntry3.setReason("Manual Redaction"); + manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build())); + + var notFoundId = UUID.randomUUID().toString(); + ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry(); + manualRedactionEntry4.setAnnotationId(notFoundId); + manualRedactionEntry4.setFileId("fileId"); + manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED); + manualRedactionEntry4.setType("CBI_author"); + manualRedactionEntry4.setValue("Random"); + manualRedactionEntry4.setReason("Manual Redaction"); + manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build())); + + manualRedactions.getEntriesToAdd().add(manualRedactionEntry); + manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); + manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); + manualRedactions.getEntriesToAdd().add(manualRedactionEntry4); + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + request.setManualRedactions(manualRedactions); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + + List unprocessedManualEntities = unprocessedChangesService.addInformationToUnprocessedManualEntities(TEST_FILE_ID, TEST_DOSSIER_ID, TEST_DOSSIER_TEMPLATE_ID, manualRedactions); + + assertFalse(unprocessedManualEntities.isEmpty()); + assertEquals(unprocessedManualEntities.size(), 4); + + Optional optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst(); + assertTrue(optionalUnprocessedManualEntity.isPresent()); + + UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get(); + assertEquals(unprocessedManualEntity.getTextBefore(), "was above the "); + assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%"); + assertEquals(unprocessedManualEntity.getStartOffset(), 2766); + assertEquals(unprocessedManualEntity.getEndOffset(), 2770); + assertEquals(unprocessedManualEntity.getClosestHeadline(), "2.6.14 Summary of product exposure and risk assessment "); + assertEquals(unprocessedManualEntity.getContainingNodeId().get(0), 1); + assertEquals(unprocessedManualEntity.getContainingNodeId().get(1), 1); + assertEquals(unprocessedManualEntity.getColor()[0], 0f); + assertEquals(unprocessedManualEntity.getColor()[1], 0f); + assertEquals(unprocessedManualEntity.getColor()[2], 0f); + + Optional optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst(); + assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent()); + + UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get(); + assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), ""); + assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), ""); + assertEquals(unprocessedNotFoundManualEntity.getStartOffset(), -1); + assertEquals(unprocessedNotFoundManualEntity.getEndOffset(), -1); + assertEquals(unprocessedNotFoundManualEntity.getClosestHeadline(), ""); + assertTrue(unprocessedNotFoundManualEntity.getContainingNodeId().isEmpty()); + assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f); + assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f); + assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f); + } +}