Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-10186 Unlinked annotation with manual changes still linked and removed in specific corner case

Browse Source
This commit is contained in:
Corina Olariu 2024-11-04 14:24:18 +01:00
parent c9468f3cf4
commit 288e0d3c51
6 changed files with 145 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java
View File

@ -95,7 +95,9 @@ public interface IEntity {
*/
// Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
default boolean applied() {
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isApplied();
}
return getManualOverwrite().getApplied()
.orElse(getMatchedRule().isApplied());
}
@ -118,6 +120,9 @@ public interface IEntity {
* @return True if ignored, false otherwise.
*/
default boolean ignored() {
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isIgnored();
}
return getManualOverwrite().getIgnored()
.orElse(getMatchedRule().isIgnored());
@ -130,8 +135,10 @@ public interface IEntity {
* @return True if removed, false otherwise.
*/
default boolean removed() {
return getManualOverwrite().getRemoved()
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isRemoved();
}
return getManualOverwrite().getRemoved()
.orElse(getMatchedRule().isRemoved());
}
@ -142,7 +149,9 @@ public interface IEntity {
* @return True if resized, false otherwise.
*/
default boolean resized() {
if (this.getMatchedRule().isHigherPriorityThanManual()) {
return getMatchedRule().isRemoved();
}
return getManualOverwrite().getResized()
.orElse(false);
}

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -28,8 +28,9 @@ public final class MatchedRule implements Comparable<MatchedRule> {
public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
public static final RuleType MANUAL_TYPE = RuleType.fromString("MAN");
public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, MANUAL_TYPE, DICTIONARY_TYPE);
RuleIdentifier ruleIdentifier;
@Builder.Default
@ -56,6 +57,10 @@ public final class MatchedRule implements Comparable<MatchedRule> {
return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build();
}
public boolean isHigherPriorityThanManual() {
return (-1 < RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type())) &&
(RULE_TYPE_PRIORITIES.indexOf(this.ruleIdentifier.type()) < RULE_TYPE_PRIORITIES.indexOf(MANUAL_TYPE));
}
/**
* Returns a modified instance of {@link MatchedRule} based on its applied status.

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -390,9 +390,6 @@ public class EntityLogCreatorService {
Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
engines.add(Engine.MANUAL);
}
return engines;
}

125
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
import static com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
@ -45,11 +46,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@ -61,6 +64,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
@ -2164,7 +2168,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
String assessment = "assessment";
dossierDictionary.get(DICTIONARY_AUTHOR).add(assessment);
reanlysisVersions.put(assessment, 1L);
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(1L);
analyzeService.reanalyze(request);
@ -2191,6 +2194,126 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
}
@Test
@SneakyThrows
@Order(3)
public void testLocalRemovalOfDictEntry() {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
String pdfFile = "files/new/test_file.pdf";
String manualDictAddId = UUID.randomUUID().toString();
String manualAddId = UUID.randomUUID().toString();
String valueToAdd = "Crandu Seku Laku Meku";
// positions=[Rectangle(topLeft=Point(x=56.8, y=527.264), width=120.96, height=15.408, page=1)],
List<Rectangle> fullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(527.264f).width(120.96f).height(15.408f).page(1).build());
ManualRedactionEntry manualDictRedactionEntry = new ManualRedactionEntry();
manualDictRedactionEntry.setAnnotationId(manualDictAddId);
manualDictRedactionEntry.setFileId("fileId");
manualDictRedactionEntry.setUser("test");
manualDictRedactionEntry.setType("CBI_author");
manualDictRedactionEntry.setRectangle(false);
manualDictRedactionEntry.setAddToDictionary(true);
manualDictRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY);
manualDictRedactionEntry.setRequestDate(OffsetDateTime.now());
manualDictRedactionEntry.setValue(valueToAdd);
manualDictRedactionEntry.setReason("Dictionary Request");
manualDictRedactionEntry.setPositions(fullPositions);
var idUsedForRemoval = "4d92d86e7d70ab9bb5c0d554bfa3c7f0";
var idRemoval = getIdRemoval(idUsedForRemoval);
AnalyzeRequest request = uploadFileToStorage(pdfFile);
Set<ManualRedactionEntry> entriesToAdd = Set.of(manualDictRedactionEntry);
Set<IdRemoval> entriesToRemove = Set.of(idRemoval);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(entriesToAdd).idsToRemove(entriesToRemove).build());
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
request.setAnalysisNumber(0);
mockDictionaryCalls(1L);
dossierDictionary.get(DICTIONARY_AUTHOR).add(valueToAdd);
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L);
reanlysisVersions.put(valueToAdd, 0L);
analyzeService.analyze(request);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(entityLog.getEntityLogEntry().size(), 4);
EntityLogEntry entityLogEntry1 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval))
.findFirst()
.get();
assertEquals(entityLogEntry1.getState(), EntryState.IGNORED);
assertEquals(entityLogEntry1.getMatchedRule(), "CBI.0.3");
assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL));
//-----------force local
var processedDate = OffsetDateTime.now();
manualDictRedactionEntry.setProcessedDate(processedDate);
idRemoval.setProcessedDate(processedDate);
//[Rectangle(topLeft=Point(x=56.8, y=528.9), width=120.96001, height=12.642, page=1)]
List<Rectangle> localfullPositions = List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(528.9f).width(120.96f).height(12.642f).page(1).build());
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setUser("test");
manualRedactionEntry.setType("CBI_author");
manualRedactionEntry.setRectangle(false);
manualRedactionEntry.setAddToDictionary(false);
manualRedactionEntry.setDictionaryEntryType(DictionaryEntryType.ENTRY);
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
manualRedactionEntry.setValue(valueToAdd);
manualRedactionEntry.setReason("Author found, removed by manual override");
manualRedactionEntry.setSection("Header: This is my test");
manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
manualRedactionEntry.setTextBefore("Lorem My Ipsum ");
manualRedactionEntry.setTextAfter("Crandu Seku Laku");
manualRedactionEntry.setPositions(localfullPositions);
ManualForceRedaction forceRequest = new ManualForceRedaction();
forceRequest.setAnnotationId(manualAddId);
forceRequest.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
forceRequest.setUser("test");
forceRequest.setRequestDate(OffsetDateTime.now());
Set<ManualForceRedaction> forceRedactions = Set.of(forceRequest);
request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualDictRedactionEntry, manualRedactionEntry))
.idsToRemove(entriesToRemove)
.forceRedactions(forceRedactions)
.build());
request.setAnalysisNumber(2);
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(entityLog.getEntityLogEntry().size(), 5);
entityLogEntry1 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(idUsedForRemoval))
.findFirst()
.get();
var entityLogEntry2 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get();
assertEquals(entityLogEntry1.getState(), EntryState.REMOVED);
assertEquals(entityLogEntry1.getMatchedRule(), "X.11.1");
assertFalse(entityLogEntry1.getEngines().contains(Engine.MANUAL));
assertEquals(entityLogEntry2.getState(), EntryState.APPLIED);
assertEquals(entityLogEntry2.getMatchedRule(), "MAN.5.0");
dossierDictionary.get(DICTIONARY_AUTHOR).remove(valueToAdd);
}
@Test
@SneakyThrows
public void testDocumentDataFallback() {

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java
View File

@ -30,8 +30,8 @@ public class TextEntityTest {
entity.apply("MAN.3.0", "");
entity.apply("MAN.3.3", "");
entity.skip("CBI.13.2", "");
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("MAN.3.0");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(3);
}

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test_file.pdf Normal file
View File

Binary file not shown.