RED-7784 - Apply unprocessed manual changes in reports and redactions
This commit is contained in:
Andrei Isvoran
parent
0ace45ca61
commit
7d0a159aed
@ -0,0 +1,24 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@EqualsAndHashCode
|
||||
public class UnprocessedManualEntity {
|
||||
|
||||
private String annotationId;
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
private int startOffset;
|
||||
private int endOffset;
|
||||
private String closestHeadline;
|
||||
private float[] color;
|
||||
private List<Integer> containingNodeId;
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.resources;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
|
||||
public interface UnprocessedManualEntityResource {
|
||||
|
||||
@PostMapping(value = "/mergeUnprocessedManualEntities", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
List<UnprocessedManualEntity> mergeUnprocessedManualEntities(@RequestParam(value = "fileId") String fileId,
|
||||
@RequestParam(value = "dossierId") String dossierId,
|
||||
@RequestParam(value = "dossierTemplateId") String dosserTemplateId,
|
||||
@RequestBody ManualRedactions manualRedactions);
|
||||
}
|
||||
@ -16,7 +16,7 @@ val layoutParserVersion = "0.74.0"
|
||||
val jacksonVersion = "2.15.2"
|
||||
val droolsVersion = "9.44.0.Final"
|
||||
val pdfBoxVersion = "3.0.0"
|
||||
val persistenceServiceVersion = "2.210.0"
|
||||
val persistenceServiceVersion = "2.223.0"
|
||||
|
||||
configurations {
|
||||
all {
|
||||
|
||||
@ -0,0 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
|
||||
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class UnprocessedManualEntityController implements UnprocessedManualEntityResource {
|
||||
|
||||
private final UnprocessedChangesService unprocessedChangesService;
|
||||
|
||||
|
||||
@Override
|
||||
public List<UnprocessedManualEntity> mergeUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
|
||||
|
||||
return unprocessedChangesService.addInformationToUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,88 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class UnprocessedChangesService {
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
private final DictionaryService dictionaryService;
|
||||
private final ManualEntityCreationService manualEntityCreationService;
|
||||
|
||||
|
||||
public List<UnprocessedManualEntity> addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
|
||||
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
|
||||
Set<String> annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
|
||||
|
||||
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntitiesConverter(manualRedactions), document);
|
||||
|
||||
document.getEntities().forEach(textEntity -> {
|
||||
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
||||
.annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
|
||||
.closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
||||
.color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
|
||||
.startOffset(textEntity.getTextRange().start())
|
||||
.endOffset(textEntity.getTextRange().end())
|
||||
.containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
|
||||
.textBefore(textEntity.getTextBefore())
|
||||
.textAfter(textEntity.getTextAfter())
|
||||
.build());
|
||||
});
|
||||
|
||||
notFoundManualEntities.forEach(manualEntity -> {
|
||||
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
||||
.annotationId(manualEntity.getId())
|
||||
.color(getColor(manualEntity.getType(), dossierTemplateId, manualEntity.applied(), manualEntity.getEntityType().equals(EntityType.HINT)))
|
||||
.closestHeadline("")
|
||||
.startOffset(-1)
|
||||
.endOffset(-1)
|
||||
.containingNodeId(Collections.emptyList())
|
||||
.textAfter("")
|
||||
.textBefore("")
|
||||
.build());
|
||||
});
|
||||
|
||||
return unprocessedManualEntities;
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
|
||||
|
||||
if (!isApplied && !isHint) {
|
||||
return dictionaryService.getNotRedactedColor(dossierTemplateId);
|
||||
}
|
||||
return dictionaryService.getColor(type, dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
|
||||
|
||||
return manualRedactions.getEntriesToAdd().stream()
|
||||
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
|
||||
manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only"))).toList();
|
||||
}
|
||||
|
||||
}
|
||||
@ -66,7 +66,7 @@ public class ManualEntityCreationService {
|
||||
}
|
||||
|
||||
|
||||
private List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
|
||||
public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
|
||||
|
||||
Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
|
||||
Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
|
||||
|
||||
@ -0,0 +1,165 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service.document;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.FilterType;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.iqser.red.commons.jackson.ObjectMapperFactory;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
|
||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
|
||||
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
|
||||
public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
@Import({LayoutParsingServiceProcessorConfiguration.class})
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public StorageService inmemoryStorage() {
|
||||
|
||||
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Autowired
|
||||
RedactionStorageService redactionStorageService;
|
||||
|
||||
@MockBean
|
||||
DictionaryService dictionaryService;
|
||||
|
||||
@Autowired
|
||||
UnprocessedChangesService unprocessedChangesService;
|
||||
|
||||
@Test
|
||||
public void testManualSurroundingText() {
|
||||
|
||||
String pdfFile = "files/new/S4.pdf";
|
||||
when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
|
||||
when(dictionaryService.getNotRedactedColor(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
|
||||
manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString());
|
||||
manualRedactionEntry.setFileId("fileId");
|
||||
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry.setType("CBI_author");
|
||||
manualRedactionEntry.setValue("rabbits");
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
|
||||
manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
|
||||
manualRedactionEntry2.setFileId("fileId");
|
||||
manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry2.setType("CBI_author");
|
||||
manualRedactionEntry2.setValue("rabbits");
|
||||
manualRedactionEntry2.setReason("Manual Redaction");
|
||||
manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
|
||||
|
||||
var aoelId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
|
||||
manualRedactionEntry3.setAnnotationId(aoelId);
|
||||
manualRedactionEntry3.setFileId("fileId");
|
||||
manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry3.setType("CBI_author");
|
||||
manualRedactionEntry3.setValue("AOEL");
|
||||
manualRedactionEntry3.setReason("Manual Redaction");
|
||||
manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
|
||||
|
||||
var notFoundId = UUID.randomUUID().toString();
|
||||
ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry();
|
||||
manualRedactionEntry4.setAnnotationId(notFoundId);
|
||||
manualRedactionEntry4.setFileId("fileId");
|
||||
manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED);
|
||||
manualRedactionEntry4.setType("CBI_author");
|
||||
manualRedactionEntry4.setValue("Random");
|
||||
manualRedactionEntry4.setReason("Manual Redaction");
|
||||
manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()));
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry4);
|
||||
|
||||
AnalyzeRequest request = uploadFileToStorage(pdfFile);
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
|
||||
|
||||
List<UnprocessedManualEntity> unprocessedManualEntities = unprocessedChangesService.addInformationToUnprocessedManualEntities(TEST_FILE_ID, TEST_DOSSIER_ID, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
|
||||
|
||||
assertFalse(unprocessedManualEntities.isEmpty());
|
||||
assertEquals(unprocessedManualEntities.size(), 4);
|
||||
|
||||
Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
|
||||
assertTrue(optionalUnprocessedManualEntity.isPresent());
|
||||
|
||||
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
|
||||
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
|
||||
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
|
||||
assertEquals(unprocessedManualEntity.getStartOffset(), 2766);
|
||||
assertEquals(unprocessedManualEntity.getEndOffset(), 2770);
|
||||
assertEquals(unprocessedManualEntity.getClosestHeadline(), "2.6.14 Summary of product exposure and risk assessment ");
|
||||
assertEquals(unprocessedManualEntity.getContainingNodeId().get(0), 1);
|
||||
assertEquals(unprocessedManualEntity.getContainingNodeId().get(1), 1);
|
||||
assertEquals(unprocessedManualEntity.getColor()[0], 0f);
|
||||
assertEquals(unprocessedManualEntity.getColor()[1], 0f);
|
||||
assertEquals(unprocessedManualEntity.getColor()[2], 0f);
|
||||
|
||||
Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
|
||||
assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
|
||||
|
||||
UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
|
||||
assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
|
||||
assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
|
||||
assertEquals(unprocessedNotFoundManualEntity.getStartOffset(), -1);
|
||||
assertEquals(unprocessedNotFoundManualEntity.getEndOffset(), -1);
|
||||
assertEquals(unprocessedNotFoundManualEntity.getClosestHeadline(), "");
|
||||
assertTrue(unprocessedNotFoundManualEntity.getContainingNodeId().isEmpty());
|
||||
assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f);
|
||||
assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f);
|
||||
assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f);
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user