diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index 3cc3cb3c..9929cc3c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -95,7 +95,9 @@ public interface IEntity { */ // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner. default boolean applied() { - + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isApplied(); + } return getManualOverwrite().getApplied() .orElse(getMatchedRule().isApplied()); } @@ -118,6 +120,9 @@ public interface IEntity { * @return True if ignored, false otherwise. */ default boolean ignored() { + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isIgnored(); + } return getManualOverwrite().getIgnored() .orElse(getMatchedRule().isIgnored()); @@ -130,8 +135,10 @@ public interface IEntity { * @return True if removed, false otherwise. */ default boolean removed() { - - return getManualOverwrite().getRemoved() + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isRemoved(); + } + return getManualOverwrite().getRemoved() .orElse(getMatchedRule().isRemoved()); } @@ -142,7 +149,9 @@ public interface IEntity { * @return True if resized, false otherwise. */ default boolean resized() { - + if (this.getMatchedRule().isHigherPriorityThanManual()) { + return getMatchedRule().isRemoved(); + } return getManualOverwrite().getResized() .orElse(false); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index a72a9adf..02f485c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -28,8 +28,9 @@ public final class MatchedRule implements Comparable { public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL"); public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP"); + public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN"); public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT"); - private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE); + private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, MANUAL_TYPE, DICTIONARY_TYPE); RuleIdentifier ruleIdentifier; @Builder.Default @@ -56,6 +57,10 @@ public final class MatchedRule implements Comparable { return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build(); } + public boolean isHigherPriorityThanManual() { + return (-1 < RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type())) && + (RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type()) < RULE_TYPE_PRIORITIES.indexOf(MANUAL_TYPE)); + } /** * Returns a modified instance of {@link MatchedRule} based on its applied status. diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 421f4e3d..380a1cf7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -390,9 +390,6 @@ public class EntityLogCreatorService { Set engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>(); - if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) { - engines.add(Engine.MANUAL); - } return engines; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 8dc9ad8a..b1eba29c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server; import static com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -45,11 +46,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; @@ -61,6 +64,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; @@ -2164,7 +2168,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { String assessment = "assessment"; dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment); reanlysisVersions.put(assessment, 1L); - when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L); analyzeService.reanalyze(request); @@ -2191,6 +2194,126 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { } + @Test + @SneakyThrows + @Order(3) + public void testLocalRemovalOfDictEntry() { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + + String pdfFile = "files/new/test_file.pdf"; + + String manualDictAddId = UUID.randomUUID().toString(); + String manualAddId = UUID.randomUUID().toString(); + String valueToAdd = "Crandu Seku Laku Meku"; + +// positions=[Rectangle(topLeft=Point(x=56.8, y=527.264), width=120.96, height=15.408, page=1)], + List fullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(527.264f).width(120.96f).height(15.408f).page(1).build()); + ManualRedactionEntry manualDictRedactionEntry = new ManualRedactionEntry(); + manualDictRedactionEntry.setAnnotationId(manualDictAddId); + manualDictRedactionEntry.setFileId("fileId"); + manualDictRedactionEntry.setUser("test"); + manualDictRedactionEntry.setType("CBI_author"); + manualDictRedactionEntry.setRectangle(false); + manualDictRedactionEntry.setAddToDictionary(true); + manualDictRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY); + manualDictRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualDictRedactionEntry.setValue(valueToAdd); + manualDictRedactionEntry.setReason("Dictionary Request"); + manualDictRedactionEntry.setPositions(fullPositions); + + var idUsedForRemoval = "4d92d86e7d70ab9bb5c0d554bfa3c7f0"; + var idRemoval = getIdRemoval(idUsedForRemoval); + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + Set entriesToAdd = Set.of(manualDictRedactionEntry); + Set entriesToRemove = Set.of(idRemoval); + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).idsToRemove(entriesToRemove).build()); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + request.setAnalysisNumber(0); + mockDictionaryCalls(1L); + + dossierDictionary.get(DICTIONARY_AUTHOR).add(valueToAdd); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L); + reanlysisVersions.put(valueToAdd, 0L); + + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertEquals(entityLog.getEntityLogEntry().size(), 4); + + EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval)) + .findFirst() + .get(); + assertEquals(entityLogEntry1.getState(), EntryState.IGNORED); + assertEquals(entityLogEntry1.getMatchedRule(), "CBI.0.3"); + assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL)); +//-----------force local + var processedDate = OffsetDateTime.now(); + manualDictRedactionEntry.setProcessedDate(processedDate); + idRemoval.setProcessedDate(processedDate); + + //[Rectangle(topLeft=Point(x=56.8, y=528.9), width=120.96001, height=12.642, page=1)] + List localfullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(528.9f).width(120.96f).height(12.642f).page(1).build()); + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(manualAddId); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setUser("test"); + manualRedactionEntry.setType("CBI_author"); + manualRedactionEntry.setRectangle(false); + manualRedactionEntry.setAddToDictionary(false); + manualRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setValue(valueToAdd); + manualRedactionEntry.setReason("Author found, removed by manual override"); + manualRedactionEntry.setSection("Header: This is my test"); + manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + manualRedactionEntry.setTextBefore("Lorem My Ipsum "); + manualRedactionEntry.setTextAfter("Crandu Seku Laku"); + manualRedactionEntry.setPositions(localfullPositions); + + ManualForceRedaction forceRequest = new ManualForceRedaction(); + forceRequest.setAnnotationId(manualAddId); + forceRequest.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + forceRequest.setUser("test"); + forceRequest.setRequestDate(OffsetDateTime.now()); + + Set forceRedactions = Set.of(forceRequest); + request.setManualRedactions(ManualRedactions.builder() + .entriesToAdd(Set.of(manualDictRedactionEntry, manualRedactionEntry)) + .idsToRemove(entriesToRemove) + .forceRedactions(forceRedactions) + .build()); + request.setAnalysisNumber(2); + + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + assertEquals(entityLog.getEntityLogEntry().size(), 5); + + entityLogEntry1 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval)) + .findFirst() + .get(); + var entityLogEntry2 = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get(); + assertEquals(entityLogEntry1.getState(), EntryState.REMOVED); + assertEquals(entityLogEntry1.getMatchedRule(), "X.11.1"); + assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL)); + assertEquals(entityLogEntry2.getState(), EntryState.APPLIED); + assertEquals(entityLogEntry2.getMatchedRule(), "MAN.5.0"); + dossierDictionary.get(DICTIONARY_AUTHOR).remove(valueToAdd); + } + + @Test @SneakyThrows public void testDocumentDataFallback() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java index 75a77785..32fb8fbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java @@ -30,8 +30,8 @@ public class TextEntityTest { entity.apply("MAN.3.0", ""); entity.apply("MAN.3.3", ""); entity.skip("CBI.13.2", ""); - assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2"); - assertThat(entity.getMatchedRuleUnit()).isEqualTo(13); + assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("MAN.3.0"); + assertThat(entity.getMatchedRuleUnit()).isEqualTo(3); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf new file mode 100644 index 00000000..8daa5ae3 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf differ