Merge branch 'RED-8043' into 'master'

RED-8043 - Add SurroundTextAnalysis case for unprocessed redactions to be...

Closes RED-8043

See merge request redactmanager/redaction-service!227
This commit is contained in:
Andrei Isvoran 2023-12-13 16:06:09 +01:00
commit aa8ed82379
10 changed files with 93 additions and 137 deletions

View File

@ -0,0 +1,18 @@
package com.iqser.red.service.redaction.v1.model;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class AnalyzeResponse {
private String fileId;
private List<UnprocessedManualEntity> unprocessedManualEntities;
}

View File

@ -3,12 +3,12 @@ package com.iqser.red.service.redaction.v1.model;
public class QueueNames {
public static final String REDACTION_QUEUE = "redactionQueue";
public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
public static final String REDACTION_DQL = "redactionDQL";
public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
public static final String MIGRATION_QUEUE = "migrationQueue";
public static final String MIGRATION_DLQ = "migrationDLQ";
public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
public static final String MIGRATION_DLQ = "migrationDLQ";
}

View File

@ -21,13 +21,7 @@ public class UnprocessedManualEntity {
private String annotationId;
private String textBefore;
private String textAfter;
private int startOffset;
private int endOffset;
private String closestHeadline;
private String section;
private String legalBasis;
private float[] color;
private List<Integer> containingNodeId;
@Builder.Default
private List<Position> positions = new ArrayList<>();
}

View File

@ -1,20 +0,0 @@
package com.iqser.red.service.redaction.v1.resources;
import java.util.List;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
public interface UnprocessedManualEntityResource {
@PostMapping(value = "/mergeUnprocessedManualEntities", produces = MediaType.APPLICATION_JSON_VALUE)
List<UnprocessedManualEntity> mergeUnprocessedManualEntities(@RequestParam(value = "fileId") String fileId,
@RequestParam(value = "dossierId") String dossierId,
@RequestParam(value = "dossierTemplateId") String dosserTemplateId,
@RequestBody ManualRedactions manualRedactions);
}

View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.75.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.242.0"
val persistenceServiceVersion = "2.270.0"
val springBootStarterVersion = "3.1.5"
configurations {

View File

@ -1,29 +0,0 @@
package com.iqser.red.service.redaction.v1.server.controller;
import java.util.List;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RestController
@RequiredArgsConstructor
public class UnprocessedManualEntityController implements UnprocessedManualEntityResource {
private final UnprocessedChangesService unprocessedChangesService;
@Override
public List<UnprocessedManualEntity> mergeUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
return unprocessedChangesService.addInformationToUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions);
}
}

View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.queue;
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
@ -69,6 +70,16 @@ public class MessagingConfiguration {
.build();
}
@Bean
public Queue redactionAnalysisResponseQueue() {
return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
.maxPriority(2)
.build();
}
@Bean
public Queue redactionDeadLetterQueue() {

View File

@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
import lombok.RequiredArgsConstructor;
@ -39,6 +40,7 @@ public class RedactionMessageReceiver {
private final AnalyzeService analyzeService;
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
private final RulesClient rulesClient;
private final UnprocessedChangesService unprocessedChangesService;
@SneakyThrows
@ -70,7 +72,7 @@ public class RedactionMessageReceiver {
log.info("");
log.info("Processing AnalyzeRequest for file: {}, priority: {} ", analyzeRequest.getFileId(), priority);
log.info("");
AnalyzeResult result;
AnalyzeResult result = new AnalyzeResult();
try {
switch (analyzeRequest.getMessageType()) {
@ -98,6 +100,16 @@ public class RedactionMessageReceiver {
format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------------------------");
break;
case SURROUNDING_TEXT_ANALYSIS:
log.info("------------------------------Surrounding Text Analysis------------------------------------------");
log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.info(analyzeRequest.getManualRedactions().toString());
unprocessedChangesService.analyseSurroundingText(analyzeRequest);
log.info("Successful Surrounding Text Analysis dossier {} file {} ",
analyzeRequest.getDossierId(),
analyzeRequest.getFileId());
log.info("-------------------------------------------------------------------------------------------------");
break;
default:
throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());

View File

@ -1,50 +1,52 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class UnprocessedChangesService {
private final RedactionStorageService redactionStorageService;
private final DictionaryService dictionaryService;
private final ManualEntityCreationService manualEntityCreationService;
RedactionStorageService redactionStorageService;
ManualEntityCreationService manualEntityCreationService;
RabbitTemplate rabbitTemplate;
public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
public List<UnprocessedManualEntity> addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
// FIXME, because of the color if you start redaction-service a no processing is performed, a nullpointer will be throw when getting color.
// The entire concept needs to be rethought, because it is bullshit.
dictionaryService.updateDictionary(dossierTemplateId, dossierId);
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
Set<String> annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
Set<String> annotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
annotationIds.addAll(analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()));
List<ManualEntity> notFoundManualEntities = new ArrayList<>();
List<ManualEntity> manualEntities = manualEntitiesConverter(manualRedactions);
List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions());
if (!manualEntities.isEmpty()) {
notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
}
@ -60,54 +62,31 @@ public class UnprocessedChangesService {
.toList();
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
.annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
.closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
.color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
.startOffset(textEntity.getTextRange().start())
.endOffset(textEntity.getTextRange().end())
.containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
.textBefore(textEntity.getTextBefore())
.textAfter(textEntity.getTextAfter())
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
.positions(positions)
.legalBasis(textEntity.legalBasis())
.build());
}
});
notFoundManualEntities.forEach(manualEntity -> {
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
.annotationId(manualEntity.getId())
.color(getColor(manualEntity.getType(), dossierTemplateId, manualEntity.applied(), manualEntity.getEntityType().equals(EntityType.HINT)))
.closestHeadline("")
.startOffset(-1)
.endOffset(-1)
.containingNodeId(Collections.emptyList())
.textAfter("")
.textBefore("")
.section("")
.positions(manualEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.build());
});
notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(UnprocessedManualEntity.builder()
.annotationId(manualEntity.getId())
.textAfter("")
.textBefore("")
.section("")
.positions(manualEntity.getManualOverwrite()
.getPositions()
.orElse(manualEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.build()));
return unprocessedManualEntities;
rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE, AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
}
private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
if (!isApplied && !isHint) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
return manualRedactions.getEntriesToAdd().stream()

View File

@ -3,7 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import java.util.List;
import java.util.Optional;
@ -11,11 +12,14 @@ import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
@ -30,13 +34,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@ -63,11 +67,11 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
}
@Autowired
RedactionStorageService redactionStorageService;
@Captor
ArgumentCaptor<AnalyzeResponse> captor;
@MockBean
DictionaryService dictionaryService;
@SpyBean
RabbitTemplate rabbitTemplate;
@Autowired
UnprocessedChangesService unprocessedChangesService;
@ -76,8 +80,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
public void testManualSurroundingText() {
String pdfFile = "files/new/S4.pdf";
when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
when(dictionaryService.getNotRedactedColor(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
ManualRedactions manualRedactions = new ManualRedactions();
@ -128,7 +130,11 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
request.setManualRedactions(manualRedactions);
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
List<UnprocessedManualEntity> unprocessedManualEntities = unprocessedChangesService.addInformationToUnprocessedManualEntities(TEST_FILE_ID, TEST_DOSSIER_ID, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
unprocessedChangesService.analyseSurroundingText(request);
verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
assertFalse(unprocessedManualEntities.isEmpty());
assertEquals(unprocessedManualEntities.size(), 4);
@ -139,15 +145,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntity.getStartOffset(), 2766);
assertEquals(unprocessedManualEntity.getEndOffset(), 2770);
assertEquals(unprocessedManualEntity.getClosestHeadline(), "2.6.14 Summary of product exposure and risk assessment ");
assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntity.getContainingNodeId().get(0), 1);
assertEquals(unprocessedManualEntity.getContainingNodeId().get(1), 1);
assertEquals(unprocessedManualEntity.getColor()[0], 0f);
assertEquals(unprocessedManualEntity.getColor()[1], 0f);
assertEquals(unprocessedManualEntity.getColor()[2], 0f);
assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
@ -159,14 +157,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
assertEquals(unprocessedNotFoundManualEntity.getStartOffset(), -1);
assertEquals(unprocessedNotFoundManualEntity.getEndOffset(), -1);
assertEquals(unprocessedNotFoundManualEntity.getClosestHeadline(), "");
assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
assertTrue(unprocessedNotFoundManualEntity.getContainingNodeId().isEmpty());
assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f);
assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f);
assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);