RED-8043 - Add SurroundTextAnalysis case for unprocessed redactions to be...

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class AnalyzeResponse {
+
+    private String fileId;
+    private List<UnprocessedManualEntity> unprocessedManualEntities;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java

@@ -3,12 +3,12 @@ package com.iqser.red.service.redaction.v1.model;
 public class QueueNames {
 
     public static final String REDACTION_QUEUE = "redactionQueue";
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+    public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
     public static final String REDACTION_DQL = "redactionDQL";
 
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-
     public static final String MIGRATION_QUEUE = "migrationQueue";
-    public static final String MIGRATION_DLQ = "migrationDLQ";
     public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
+    public static final String MIGRATION_DLQ = "migrationDLQ";
 
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -21,13 +21,7 @@ public class UnprocessedManualEntity {
     private String annotationId;
     private String textBefore;
     private String textAfter;
-    private int startOffset;
-    private int endOffset;
-    private String closestHeadline;
     private String section;
-    private String legalBasis;
-    private float[] color;
-    private List<Integer> containingNodeId;
     @Builder.Default
     private List<Position> positions = new ArrayList<>();
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/UnprocessedManualEntityResource.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.resources;
-
-import java.util.List;
-
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestParam;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
-
-public interface UnprocessedManualEntityResource {
-
-    @PostMapping(value = "/mergeUnprocessedManualEntities", produces = MediaType.APPLICATION_JSON_VALUE)
-    List<UnprocessedManualEntity> mergeUnprocessedManualEntities(@RequestParam(value = "fileId") String fileId,
-                                                                 @RequestParam(value = "dossierId") String dossierId,
-                                                                 @RequestParam(value = "dossierTemplateId") String dosserTemplateId,
-                                                                 @RequestBody ManualRedactions manualRedactions);
-}

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,7 +16,7 @@ val layoutParserVersion = "0.75.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.242.0"
+val persistenceServiceVersion = "2.270.0"
 val springBootStarterVersion = "3.1.5"
 
 configurations {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/UnprocessedManualEntityController.java

@@ -1,29 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.controller;
-
-import java.util.List;
-
-import org.springframework.web.bind.annotation.RestController;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
-import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
-import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@RestController
-@RequiredArgsConstructor
-public class UnprocessedManualEntityController implements UnprocessedManualEntityResource {
-
-    private final UnprocessedChangesService unprocessedChangesService;
-
-
-    @Override
-    public List<UnprocessedManualEntity> mergeUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
-
-        return unprocessedChangesService.addInformationToUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.queue;
 
 import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
 import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE;
 import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
 import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
 import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
@@ -69,6 +70,16 @@ public class MessagingConfiguration {
                 .build();
     }
 
+    @Bean
+    public Queue redactionAnalysisResponseQueue() {
+
+        return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
+                .maxPriority(2)
+                .build();
+    }
+
 
     @Bean
     public Queue redactionDeadLetterQueue() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -24,6 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import lombok.RequiredArgsConstructor;
@@ -39,6 +40,7 @@ public class RedactionMessageReceiver {
     private final AnalyzeService analyzeService;
     private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
     private final RulesClient rulesClient;
+    private final UnprocessedChangesService unprocessedChangesService;
 
 
     @SneakyThrows
@@ -70,7 +72,7 @@ public class RedactionMessageReceiver {
         log.info("");
         log.info("Processing AnalyzeRequest for file: {},  priority: {} ", analyzeRequest.getFileId(), priority);
         log.info("");
-        AnalyzeResult result;
+        AnalyzeResult result = new AnalyzeResult();
 
         try {
             switch (analyzeRequest.getMessageType()) {
@@ -98,6 +100,16 @@ public class RedactionMessageReceiver {
                             format("%.2f", result.getDuration() / 1000.0));
                     log.info("----------------------------------------------------------------------------------");
                     break;
+                case SURROUNDING_TEXT_ANALYSIS:
+                    log.info("------------------------------Surrounding Text Analysis------------------------------------------");
+                    log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+                    log.info(analyzeRequest.getManualRedactions().toString());
+                    unprocessedChangesService.analyseSurroundingText(analyzeRequest);
+                    log.info("Successful Surrounding Text Analysis dossier {} file {} ",
+                            analyzeRequest.getDossierId(),
+                            analyzeRequest.getFileId());
+                    log.info("-------------------------------------------------------------------------------------------------");
+                    break;
 
                 default:
                     throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -1,50 +1,52 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
+import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class UnprocessedChangesService {
 
-    private final RedactionStorageService redactionStorageService;
+    RedactionStorageService redactionStorageService;
-    private final DictionaryService dictionaryService;
+    ManualEntityCreationService manualEntityCreationService;
-    private final ManualEntityCreationService manualEntityCreationService;
+    RabbitTemplate rabbitTemplate;
 
+    public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
 
-    public List<UnprocessedManualEntity> addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
-
-        // FIXME, because of the color if you start redaction-service a no processing is performed, a nullpointer will be throw when getting color.
-        // The entire concept needs to be rethought, because it is bullshit.
-        dictionaryService.updateDictionary(dossierTemplateId, dossierId);
-
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
         List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
-        Set<String> annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        Set<String> annotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
+        annotationIds.addAll(analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()));
 
         List<ManualEntity> notFoundManualEntities = new ArrayList<>();
-        List<ManualEntity> manualEntities = manualEntitiesConverter(manualRedactions);
+        List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions());
         if (!manualEntities.isEmpty()) {
             notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
         }
@@ -60,54 +62,31 @@ public class UnprocessedChangesService {
                         .toList();
                 unprocessedManualEntities.add(UnprocessedManualEntity.builder()
                         .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
-                        .closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
-                        .color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
-                        .startOffset(textEntity.getTextRange().start())
-                        .endOffset(textEntity.getTextRange().end())
-                        .containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
                         .textBefore(textEntity.getTextBefore())
                         .textAfter(textEntity.getTextAfter())
                         .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
                         .positions(positions)
-                        .legalBasis(textEntity.legalBasis())
                         .build());
             }
-
         });
 
-        notFoundManualEntities.forEach(manualEntity -> {
+        notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(UnprocessedManualEntity.builder()
-            unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                .annotationId(manualEntity.getId())
-                    .annotationId(manualEntity.getId())
+                .textAfter("")
-                    .color(getColor(manualEntity.getType(), dossierTemplateId, manualEntity.applied(), manualEntity.getEntityType().equals(EntityType.HINT)))
+                .textBefore("")
-                    .closestHeadline("")
+                .section("")
-                    .startOffset(-1)
+                .positions(manualEntity.getManualOverwrite()
-                    .endOffset(-1)
+                        .getPositions()
-                    .containingNodeId(Collections.emptyList())
+                        .orElse(manualEntity.getEntityPosition())
-                    .textAfter("")
+                        .stream()
-                    .textBefore("")
+                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                    .section("")
+                        .toList())
-                    .positions(manualEntity.getManualOverwrite()
+                .build()));
-                            .getPositions()
-                            .orElse(manualEntity.getEntityPosition())
-                            .stream()
-                            .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                            .toList())
-                    .build());
-        });
 
-        return unprocessedManualEntities;
+
+        rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE, AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
     }
 
-
-    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
-
-        if (!isApplied && !isHint) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-
     private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
 
         return manualRedactions.getEntriesToAdd().stream()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -3,7 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service.document;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.when;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
 
 import java.util.List;
 import java.util.Optional;
@@ -11,11 +12,14 @@ import java.util.UUID;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.boot.test.mock.mockito.SpyBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
@@ -30,13 +34,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
+import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@@ -63,11 +67,11 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
     }
 
-    @Autowired
+    @Captor
-    RedactionStorageService redactionStorageService;
+    ArgumentCaptor<AnalyzeResponse> captor;
 
-    @MockBean
+    @SpyBean
-    DictionaryService dictionaryService;
+    RabbitTemplate rabbitTemplate;
 
     @Autowired
     UnprocessedChangesService unprocessedChangesService;
@@ -76,8 +80,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
     public void testManualSurroundingText() {
 
         String pdfFile = "files/new/S4.pdf";
-        when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
-        when(dictionaryService.getNotRedactedColor(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
 
         ManualRedactions manualRedactions = new ManualRedactions();
 
@@ -128,7 +130,11 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         request.setManualRedactions(manualRedactions);
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
 
-        List<UnprocessedManualEntity> unprocessedManualEntities = unprocessedChangesService.addInformationToUnprocessedManualEntities(TEST_FILE_ID, TEST_DOSSIER_ID, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
+        unprocessedChangesService.analyseSurroundingText(request);
+
+        verify(rabbitTemplate).convertAndSend(eq(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE), captor.capture());
+
+        List<UnprocessedManualEntity> unprocessedManualEntities = captor.getValue().getUnprocessedManualEntities();
 
         assertFalse(unprocessedManualEntities.isEmpty());
         assertEquals(unprocessedManualEntities.size(), 4);
@@ -139,15 +145,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
         assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
         assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
-        assertEquals(unprocessedManualEntity.getStartOffset(), 2766);
-        assertEquals(unprocessedManualEntity.getEndOffset(), 2770);
-        assertEquals(unprocessedManualEntity.getClosestHeadline(), "2.6.14 Summary of product exposure and risk assessment ");
         assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
-        assertEquals(unprocessedManualEntity.getContainingNodeId().get(0), 1);
-        assertEquals(unprocessedManualEntity.getContainingNodeId().get(1), 1);
-        assertEquals(unprocessedManualEntity.getColor()[0], 0f);
-        assertEquals(unprocessedManualEntity.getColor()[1], 0f);
-        assertEquals(unprocessedManualEntity.getColor()[2], 0f);
         assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
         assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
         assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
@@ -159,14 +157,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
         assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
         assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
-        assertEquals(unprocessedNotFoundManualEntity.getStartOffset(), -1);
-        assertEquals(unprocessedNotFoundManualEntity.getEndOffset(), -1);
-        assertEquals(unprocessedNotFoundManualEntity.getClosestHeadline(), "");
         assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
-        assertTrue(unprocessedNotFoundManualEntity.getContainingNodeId().isEmpty());
-        assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f);
-        assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f);
-        assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f);
         assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
         assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
         assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);