Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-9206' into 'master'

Browse Source 
RED-9206 - Sections are no longer correctly separated from each other in the test file

Closes RED-9206

See merge request redactmanager/redaction-service!415
This commit is contained in:
Corina Olariu 2024-06-04 17:13:34 +02:00
commit 758bd84a36
14 changed files with 81 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -1454,6 +1454,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

56
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -85,7 +85,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
@ -170,6 +170,60 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
}
@Test
public void testLocalPublishedRemovalWithEntityContainedByEntityOfSameType() throws IOException {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("Finished analysis");
dictionary.get(PUBLISHED_INFORMATION_INDICATOR).add("Press");
reanlysisVersions.put("Press", 2L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
mockDictionaryCalls(0L);
analyzeService.reanalyze(request);
EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
.orElseThrow();
assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
.orElseThrow();
assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
request.setManualRedactions(manualRedactions);
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
.orElseThrow();
var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.APPLIED, asyaLyon2.getState());
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
String outputFileName = OsUtils.getTemporaryDirectory() + "/AcceptanceTest.pdf";
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
}
private Stream<EntityLogEntry> findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, List<Integer> sectionNumber) {
return redactionLog.getEntityLogEntry()

17
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -188,6 +188,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.addToAllDossiers(false)
.updateDictionary(false)
.requestDate(OffsetDateTime.now())
.user("user")
.build());
request.setManualRedactions(manualRedactions);
analyzeService.reanalyze(request);
@ -233,17 +234,19 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
System.out.println("testManualRedaction");
long start = System.currentTimeMillis();
String pdfFile = "files/syngenta/CustomeFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf";
String pdfFile = "files/syngenta/CustomerFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf";
ManualRedactions manualRedactions = new ManualRedactions();
String manualAddId = UUID.randomUUID().toString();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").requestDate(OffsetDateTime.now()).build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.user("user")
.requestDate(OffsetDateTime.now())
.build()));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
@ -262,7 +265,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
AnalyzeResult result = analyzeService.analyze(request);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
@ -361,6 +364,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.type("vertebrate")
.annotationId(oxfordUniversityPress.getId())
.fileId(TEST_FILE_ID)
.user("user")
.build();
request.setManualRedactions(new ManualRedactions());
@ -409,11 +413,14 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.annotationId(annotationId)
.requestDate(OffsetDateTime.now())
.type("manual")
.user("user")
.value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
.positions(List.of(//
new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
.user("user")
.fileId(request.getFileId())
.build()));
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(annotationId)
@ -421,6 +428,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.value("Expand to Hint")
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
.updateDictionary(false)
.user("user")
.fileId(request.getFileId())
.build();
manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
request.setManualRedactions(manualRedactions);
@ -464,6 +473,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.value("Image")
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
.updateDictionary(true)
.fileId(request.getFileId())
.user("user")
.build();
manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
request.setManualRedactions(manualRedactions);

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -1242,6 +1242,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -1895,6 +1895,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1309,6 +1309,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -801,6 +801,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1330,6 +1330,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -253,6 +253,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -379,6 +379,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -279,6 +279,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -628,6 +628,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !resized(), active())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1890,6 +1890,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end

1
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1454,6 +1454,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end