Merge branch 'RED-8173-fix' into 'master'

RED-8173 - Fix for removing and adding redaction in the same place

Closes RED-8173

See merge request redactmanager/redaction-service!243

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -83,7 +83,7 @@ tasks.test {
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
-    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
+    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8")
 
     imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
     if (project.hasProperty("buildbootDockerHostNetwork")) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -77,6 +77,8 @@ public class UnprocessedChangesService {
 
         List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
 
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
         EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -11,6 +11,9 @@ import java.util.Set;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
@@ -44,11 +47,14 @@ public class ManualEntityCreationService {
     }
 
 
-    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
+    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions,
                                                                                      SemanticNode node,
                                                                                      String dossierTemplateId) {
 
-        List<ManualEntity> manualEntities = manualRedactionEntries.stream()
+        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
+        List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd().stream()
+                .filter(manualRedactionEntry -> !(idRemovals.stream().map(BaseAnnotation::getAnnotationId).toList().contains(manualRedactionEntry.getAnnotationId()) &&
+                        manualRedactionEntry.getRequestDate().isBefore(idRemovals.stream().filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())).findFirst().get().getRequestDate())))
                 .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
                 .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
                         dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java

@@ -30,8 +30,8 @@ public class ManualRedactionEntryService {
 
         List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
         if (analyzeRequest.getManualRedactions() != null) {
-            notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions()
-                    .getEntriesToAdd(), document, dossierTemplateId);
+            notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
+                    document, dossierTemplateId);
             log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         }
         if (notFoundManualRedactionEntries.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.when;
@@ -1183,4 +1184,98 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         assertThat(values).contains("Mr. Tambourine Man");
     }
 
+
+    @Test
+    @SneakyThrows
+    public void testRemovingAndAddingRedactionOnTheSameValue() {
+
+        String pdfFile = "files/new/test1S1T1.pdf";
+
+        String manualAddId = UUID.randomUUID().toString();
+        String manualAddId2 = UUID.randomUUID().toString();
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build());
+        ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId, positions, "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
+        ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2, positions, "commercial information revealing sourcing, market shares or business strategy of the applicant");
+
+        IdRemoval idRemoval = getIdRemoval(manualAddId);
+        IdRemoval idRemoval2 = getIdRemoval(manualAddId2);
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
+
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
+
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED);
+
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build());
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
+
+        manualRedactionEntry.setRequestDate(OffsetDateTime.now());
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build());
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
+    }
+
+
+    private IdRemoval getIdRemoval(String id) {
+
+        return IdRemoval.builder()
+                .annotationId(id)
+                .removeFromAllDossiers(false)
+                .removeFromDictionary(false)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.now())
+                .build();
+    }
+
+
+    private ManualRedactionEntry getManualRedactionEntry(String id, List<Rectangle> positions, String reason) {
+
+        ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
+        manualRedactionEntry2.setAnnotationId(id);
+        manualRedactionEntry2.setFileId("fileId");
+        manualRedactionEntry2.setType("manual");
+        manualRedactionEntry2.setRectangle(false);
+        manualRedactionEntry2.setRequestDate(OffsetDateTime.now());
+        manualRedactionEntry2.setValue("assessment");
+        manualRedactionEntry2.setLegalBasis("Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)");
+        manualRedactionEntry2.setReason(reason);
+        manualRedactionEntry2.setPositions(positions);
+        return manualRedactionEntry2;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java

@@ -19,6 +19,7 @@ import org.springframework.boot.test.mock.mockito.MockBean;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
@@ -121,7 +122,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
 
         assertTrue(document.getEntities().isEmpty());
 
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
                 document,
                 TEST_DOSSIER_TEMPLATE_ID);
         assertEquals(1, notFoundManualEntities.size());
@@ -171,7 +172,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
         tempEntity.removeFromGraph();
         assertTrue(document.getEntities().isEmpty());
 
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
                 document,
                 TEST_DOSSIER_TEMPLATE_ID);
         assertTrue(notFoundManualEntities.isEmpty());