Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-9139: move document to its own module, add TableOfContents and TableOfContentsItem

Browse Source 
* update LayoutEngine package
This commit is contained in:
Kilian Schuettler 2024-11-12 16:19:17 +01:00
parent 192910be9c
commit f1ba80fff6
21 changed files with 111 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentDataMapper.java
View File

@ -118,8 +118,12 @@ public class DocumentDataMapper {
.map(Integer::longValue)
.toList())
.putAllProperties(properties);
if (entry.getNode() != null) {
documentBuilder.addAllEngines(entry.getNode().getEngines());
documentBuilder.addAllEngines(entry.getNode().getEngines()
.stream()
.map(engine -> LayoutEngineProto.LayoutEngine.valueOf(engine.name()))
.toList());
} else {
documentBuilder.addAllEngines(new HashSet<>(Set.of(LayoutEngineProto.LayoutEngine.ALGORITHM)));
}

6
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentGraphMapper.java
View File

@ -20,6 +20,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
@ -99,6 +100,11 @@ public class DocumentGraphMapper {
List<Integer> treeId = entryData.getTreeIdList();
node.setTreeId(treeId);
entryData.getEnginesList()
.stream()
.map(engine -> LayoutEngine.valueOf(engine.name()))
.forEach(node::addEngine);
newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
}
return newEntries;

1
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java
View File

@ -6,7 +6,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;

7
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java Normal file
View File

@ -0,0 +1,7 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
public enum LayoutEngine {
ALGORITHM,
AI,
OUTLINE
}

1
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java
View File

@ -14,7 +14,6 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;

1
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java
View File

@ -13,7 +13,6 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;

33
redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngineMappingTest.java Normal file
View File

@ -0,0 +1,33 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto;
public class LayoutEngineMappingTest {
@Test
public void assertAllValuesMatch() {
for (LayoutEngine value : LayoutEngine.values()) {
var engine = LayoutEngineProto.LayoutEngine.valueOf(value.name());
assertEquals(engine.name(), value.name());
}
}
@Test
public void assertAllValuesMatchReverse() {
for (LayoutEngineProto.LayoutEngine value : LayoutEngineProto.LayoutEngine.values()) {
if (value.equals(LayoutEngineProto.LayoutEngine.UNRECOGNIZED)) {
continue;
}
var engine = LayoutEngine.valueOf(value.name());
assertEquals(engine.name(), value.name());
}
}
}

33
redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeTypeMappingTest.java Normal file
View File

@ -0,0 +1,33 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto;
public class NodeTypeMappingTest {
@Test
public void assertAllValuesMatch() {
for (NodeType value : NodeType.values()) {
var engine = NodeTypeProto.NodeType.valueOf(value.name());
assertEquals(engine.name(), value.name());
}
}
@Test
public void assertAllValuesMatchReverse() {
for (NodeTypeProto.NodeType value : NodeTypeProto.NodeType.values()) {
if (value.equals(NodeTypeProto.NodeType.UNRECOGNIZED)) {
continue;
}
var engine = NodeType.valueOf(value.name());
assertEquals(engine.name(), value.name());
}
}
}

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -273,6 +273,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
$table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end
// Rule unit: CBI.16
rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
when
@ -1291,7 +1292,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1430,7 +1430,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

31
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -340,7 +340,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end
// Rule unit: CBI.12 - table rules remains
// Rule unit: CBI.12
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
@ -404,8 +404,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -422,8 +420,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
});
end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -440,7 +436,6 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
});
end
rule "CBI.12.6: Skip Addresses on TableCell with header 'Owner'"
salience -1
when
@ -535,7 +530,7 @@ rule "CBI.12.15: Redacted because table row contains a vertebrate, a no_redactio
end
// Rule unit: CBI.13 - section rules
// Rule unit: CBI.13
rule "CBI.13.0: Ignore CBI Address recommendations"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -545,7 +540,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
retract($entity)
end
// from CBI.3.0
rule "CBI.13.1: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -561,7 +555,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
});
end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -570,8 +563,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -589,8 +580,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
});
end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -612,7 +601,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
});
end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -628,6 +616,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
});
end
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
@ -742,6 +731,7 @@ rule "CBI.16.3: Do not redact PII if published information found in same table r
$pii.skipWithReferences("CBI.16.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii));
end
// Rule unit: CBI.17
rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
when
@ -847,7 +837,6 @@ rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
// Rule unit: CBI.21
// from CBI.6
rule "CBI.21.0: Do not redact Names and Addresses if published_information is found in Section"
when
$section: Section(!hasTables(),
@ -882,7 +871,6 @@ rule "CBI.21.1: Do not redact Names and Addresses if published_information is fo
});
end
rule "CBI.21.2: Redact short Authors section (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1397,7 +1385,6 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
end
// UPDATED WITH LIMIT
// Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
@ -1444,7 +1431,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1475,8 +1461,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.12
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1591,7 +1575,6 @@ rule "ETC.3.3: Redact logos"
$logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
end
// from preGFL Knoell
rule "ETC.3.4: Skip logos"
when
$logo: Image(imageType == ImageType.LOGO)
@ -1759,6 +1742,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
@ -1885,7 +1869,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -2075,7 +2059,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -2214,7 +2197,6 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
retract($dictionaryEntity);
end
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
@ -2225,7 +2207,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1419,7 +1419,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when

7
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -139,6 +139,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
$entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: CBI.1
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
@ -439,7 +440,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -599,7 +599,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -708,6 +707,7 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi
retract($dossierRedaction);
end
// Rule unit: ETC.12
rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
when
@ -725,6 +725,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
@ -1018,7 +1019,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1157,7 +1157,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -149,7 +149,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -339,7 +339,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -478,7 +477,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

21
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -85,6 +85,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
@ -177,7 +178,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end
// Rule unit: CBI.12 - table rules remains
// Rule unit: CBI.12
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
@ -241,8 +242,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -259,8 +258,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
});
end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -278,8 +275,7 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
end
// Rule unit: CBI.13 - section rules
// from CBI.3.0
// Rule unit: CBI.13
rule "CBI.13.1: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -295,7 +291,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
});
end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -304,8 +299,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -323,8 +316,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
});
end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -346,7 +337,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
});
end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -362,6 +352,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
});
end
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
@ -1133,7 +1124,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1323,7 +1314,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1462,7 +1452,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -397,7 +397,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -536,7 +535,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -489,7 +489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -639,7 +638,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

26
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -389,31 +389,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -563,7 +538,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

22
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java
View File

@ -4,13 +4,11 @@ import java.io.File;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
import com.knecon.fforesight.utility.rules.management.models.BasicRule;
import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
import lombok.SneakyThrows;
@ -23,7 +21,6 @@ import lombok.experimental.UtilityClass;
@UtilityClass
public class RuleFileMigrator {
@SneakyThrows
public void migrateFile(File ruleFile) {
@ -53,21 +50,12 @@ public class RuleFileMigrator {
private static void replaceRuleIdentifiers(RuleFileBluePrint combinedBluePrint, RuleFileBluePrint ruleFileBluePrint) {
Map<String, String> identifierReplaceMap = Map.of("CBI.7.0", "CBI.16.0", "CBI.7.1", "CBI.16.1", "CBI.7.2", "CBI.16.2", "CBI.7.3", "CBI.16.3");
for (String identifier : identifierReplaceMap.keySet()) {
RuleIdentifier ruleId = RuleIdentifier.fromString(identifier);
RuleIdentifier otherRuleId = RuleIdentifier.fromString(identifierReplaceMap.get(identifier));
List<BasicRule> rulesToAdd = combinedBluePrint.findRuleByIdentifier(otherRuleId);
List<BasicRule> otherRulesToAdd = combinedBluePrint.findRuleByIdentifier(ruleId);
boolean removeRules = ruleFileBluePrint.removeRule(ruleId);
boolean removeOtherRules = ruleFileBluePrint.removeRule(otherRuleId);
if (removeRules) {
rulesToAdd.forEach(ruleFileBluePrint::addRule);
}
if (removeOtherRules) {
otherRulesToAdd.forEach(ruleFileBluePrint::addRule);
for (BasicRule rule : combinedBluePrint.getAllRules()) {
if (ruleFileBluePrint.findRuleByIdentifier(rule.identifier()).isEmpty()) {
continue;
}
ruleFileBluePrint.removeRule(rule.identifier());
ruleFileBluePrint.addRule(rule);
}
}

3
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

3
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
@ -47,8 +48,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService

12
redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java
View File

@ -5,7 +5,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator;
@ -25,17 +24,14 @@ public class RuleFileMigrationTest {
// Put your redaction service drools paths and dossier-templates paths both RM and DM here
static final List<String> ruleFileDirs = List.of(
//"/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
// "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2"
"/Users/maverickstuder/Documents/PM"
);
"/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
"/home/kschuettler/iqser/redaction/dossier-templates-v2",
"/home/kschuettler/iqser/fforesight/dossier-templates-v2");
@Test
@SneakyThrows
@Disabled
// @Disabled
void migrateAllEntityRules() {
for (String ruleFileDir : ruleFileDirs) {