From f1ba80fff6c02fe5c48dcbb0636d9ef85e243c52 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 12 Nov 2024 16:19:17 +0100 Subject: [PATCH] RED-9139: move document to its own module, add TableOfContents and TableOfContentsItem * update LayoutEngine package --- .../v1/server/mapper/DocumentDataMapper.java | 6 +++- .../v1/server/mapper/DocumentGraphMapper.java | 6 ++++ .../document/nodes/AbstractSemanticNode.java | 1 - .../model/document/nodes/LayoutEngine.java | 7 ++++ .../model/document/nodes/SemanticNode.java | 1 - .../v1/server/model/document/nodes/Table.java | 1 - .../nodes/LayoutEngineMappingTest.java | 33 +++++++++++++++++++ .../document/nodes/NodeTypeMappingTest.java | 33 +++++++++++++++++++ .../resources/drools/acceptance_rules.drl | 3 +- .../drools/all_redact_manager_rules.drl | 31 ++++------------- .../test/resources/drools/documine_flora.drl | 1 - .../resources/drools/efsa_sanitisation.drl | 7 ++-- .../drools/manual_redaction_rules.drl | 4 +-- .../src/test/resources/drools/rules.drl | 21 +++--------- .../src/test/resources/drools/rules_v2.drl | 2 -- .../src/test/resources/drools/table_demo.drl | 2 -- .../src/test/resources/drools/test_rules.drl | 26 --------------- .../migration/RuleFileMigrator.java | 22 +++---------- .../resources/all_redact_manager_rules.drl | 3 +- .../src/main/resources/all_rules_documine.drl | 3 +- .../management/RuleFileMigrationTest.java | 12 +++---- 21 files changed, 111 insertions(+), 114 deletions(-) create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java create mode 100644 redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngineMappingTest.java create mode 100644 redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeTypeMappingTest.java diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentDataMapper.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentDataMapper.java index 7410a256..419adff1 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentDataMapper.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentDataMapper.java @@ -118,8 +118,12 @@ public class DocumentDataMapper { .map(Integer::longValue) .toList()) .putAllProperties(properties); + if (entry.getNode() != null) { - documentBuilder.addAllEngines(entry.getNode().getEngines()); + documentBuilder.addAllEngines(entry.getNode().getEngines() + .stream() + .map(engine -> LayoutEngineProto.LayoutEngine.valueOf(engine.name())) + .toList()); } else { documentBuilder.addAllEngines(new HashSet<>(Set.of(LayoutEngineProto.LayoutEngine.ALGORITHM))); } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentGraphMapper.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentGraphMapper.java index 1034eec7..3d1c8136 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/mapper/DocumentGraphMapper.java @@ -20,6 +20,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; @@ -99,6 +100,11 @@ public class DocumentGraphMapper { List treeId = entryData.getTreeIdList(); node.setTreeId(treeId); + entryData.getEnginesList() + .stream() + .map(engine -> LayoutEngine.valueOf(engine.name())) + .forEach(node::addEngine); + newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build()); } return newEntries; diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java index b969d6e6..ddf32c06 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java new file mode 100644 index 00000000..3bab25ef --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngine.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.redaction.v1.server.model.document.nodes; + +public enum LayoutEngine { + ALGORITHM, + AI, + OUTLINE +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java index 75519db6..c8b33c3d 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java @@ -14,7 +14,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java index 1393af26..6e5e03a9 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; diff --git a/redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngineMappingTest.java b/redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngineMappingTest.java new file mode 100644 index 00000000..8decb157 --- /dev/null +++ b/redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/LayoutEngineMappingTest.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.redaction.v1.server.model.document.nodes; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto; + +public class LayoutEngineMappingTest { + + @Test + public void assertAllValuesMatch() { + + for (LayoutEngine value : LayoutEngine.values()) { + var engine = LayoutEngineProto.LayoutEngine.valueOf(value.name()); + assertEquals(engine.name(), value.name()); + } + } + + + @Test + public void assertAllValuesMatchReverse() { + + for (LayoutEngineProto.LayoutEngine value : LayoutEngineProto.LayoutEngine.values()) { + if (value.equals(LayoutEngineProto.LayoutEngine.UNRECOGNIZED)) { + continue; + } + var engine = LayoutEngine.valueOf(value.name()); + assertEquals(engine.name(), value.name()); + } + } + +} diff --git a/redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeTypeMappingTest.java b/redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeTypeMappingTest.java new file mode 100644 index 00000000..b20d6341 --- /dev/null +++ b/redaction-service-v1/document/src/test/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeTypeMappingTest.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.redaction.v1.server.model.document.nodes; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto; + +public class NodeTypeMappingTest { + + @Test + public void assertAllValuesMatch() { + + for (NodeType value : NodeType.values()) { + var engine = NodeTypeProto.NodeType.valueOf(value.name()); + assertEquals(engine.name(), value.name()); + } + } + + + @Test + public void assertAllValuesMatchReverse() { + + for (NodeTypeProto.NodeType value : NodeTypeProto.NodeType.values()) { + if (value.equals(NodeTypeProto.NodeType.UNRECOGNIZED)) { + continue; + } + var engine = NodeType.valueOf(value.name()); + assertEquals(engine.name(), value.name()); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index eb0b93b5..dfba8f59 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -273,6 +273,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end + // Rule unit: CBI.16 rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables" when @@ -1291,7 +1292,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -1430,7 +1430,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 0e8ab237..f576f7a3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -340,7 +340,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study end -// Rule unit: CBI.12 - table rules remains +// Rule unit: CBI.12 rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" agenda-group "LOCAL_DICTIONARY_ADDS" when @@ -404,8 +404,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); end - -//from CBI.3.3 rule "CBI.12.4: Redacted because table row contains a redaction_indicator" when $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) @@ -422,8 +420,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator" }); end - -//from CBI.3.1 rule "CBI.12.5: Redacted because table row contains a vertebrate" when $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) @@ -440,7 +436,6 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate" }); end - rule "CBI.12.6: Skip Addresses on TableCell with header 'Owner'" salience -1 when @@ -535,7 +530,7 @@ rule "CBI.12.15: Redacted because table row contains a vertebrate, a no_redactio end -// Rule unit: CBI.13 - section rules +// Rule unit: CBI.13 rule "CBI.13.0: Ignore CBI Address recommendations" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -545,7 +540,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations" retract($entity) end -// from CBI.3.0 rule "CBI.13.1: Redacted because Section contains a vertebrate" when $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -561,7 +555,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate" }); end -//from CBI.3.2 rule "CBI.13.2: Do not redact because Section does not contain a vertebrate" when $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -570,8 +563,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate" .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found")); end - -// from CBI.4.0 rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section" when $section: Section(!hasTables(), @@ -589,8 +580,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction }); end - -// from CBI.5.0 rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section" when $section: Section(!hasTables(), @@ -612,7 +601,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica }); end -// From CBI.8.0 rule "CBI.13.5: Redacted because Section contains must_redact entity" when $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -628,6 +616,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity" }); end + // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when @@ -742,6 +731,7 @@ rule "CBI.16.3: Do not redact PII if published information found in same table r $pii.skipWithReferences("CBI.16.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii)); end + // Rule unit: CBI.17 rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon" when @@ -847,7 +837,6 @@ rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC // Rule unit: CBI.21 -// from CBI.6 rule "CBI.21.0: Do not redact Names and Addresses if published_information is found in Section" when $section: Section(!hasTables(), @@ -882,7 +871,6 @@ rule "CBI.21.1: Do not redact Names and Addresses if published_information is fo }); end - rule "CBI.21.2: Redact short Authors section (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -1397,7 +1385,6 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study end -// UPDATED WITH LIMIT // Rule unit: PII.9 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" when @@ -1444,7 +1431,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end - rule "PII.10.1: Redact study director abbreviation (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -1475,8 +1461,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix" .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end - -// Rule unit: PII.12 rule "PII.12.1: Expand PII entities with salutation prefix" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -1591,7 +1575,6 @@ rule "ETC.3.3: Redact logos" $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"); end -// from preGFL Knoell rule "ETC.3.4: Skip logos" when $logo: Image(imageType == ImageType.LOGO) @@ -1759,6 +1742,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)" $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found"); end + //------------------------------------ AI rules ------------------------------------ // Rule unit: AI.0 @@ -1885,7 +1869,7 @@ rule "AI.7.0: Add all NER Entities of type Address" end -//------------------------------------ Manual redaction rules ------------------------------------ +//------------------------------------ Manual changes rules ------------------------------------ // Rule unit: MAN.0 rule "MAN.0.0: Apply manual resize redaction" @@ -2075,7 +2059,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -2214,7 +2197,6 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity" retract($dictionaryEntity); end - rule "X.11.1: Remove non manual entity which intersects with a manual entity" salience 64 when @@ -2225,7 +2207,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 77d6bf13..20a34fe2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1419,7 +1419,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index a3fd859f..f582d52a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -139,6 +139,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end + // Rule unit: CBI.1 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when @@ -439,7 +440,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end - rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -599,7 +599,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end - rule "PII.10.1: Redact study director abbreviation (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -708,6 +707,7 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi retract($dossierRedaction); end + // Rule unit: ETC.12 rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)" when @@ -725,6 +725,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)" $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found"); end + //------------------------------------ AI rules ------------------------------------ // Rule unit: AI.0 @@ -1018,7 +1019,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -1157,7 +1157,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 03612b29..5d8973cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -149,7 +149,7 @@ rule "AI.7.0: Add all NER Entities of type Address" end -//------------------------------------ Manual redaction rules ------------------------------------ +//------------------------------------ Manual changes rules ------------------------------------ // Rule unit: MAN.0 rule "MAN.0.0: Apply manual resize redaction" @@ -339,7 +339,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -478,7 +477,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index a74133d3..cf027efb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -85,6 +85,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" //------------------------------------ CBI rules ------------------------------------ + // Rule unit: CBI.0 rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx" agenda-group "LOCAL_DICTIONARY_ADDS" @@ -177,7 +178,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study end -// Rule unit: CBI.12 - table rules remains +// Rule unit: CBI.12 rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" agenda-group "LOCAL_DICTIONARY_ADDS" when @@ -241,8 +242,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); end - -//from CBI.3.3 rule "CBI.12.4: Redacted because table row contains a redaction_indicator" when $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) @@ -259,8 +258,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator" }); end - -//from CBI.3.1 rule "CBI.12.5: Redacted because table row contains a vertebrate" when $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) @@ -278,8 +275,7 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate" end -// Rule unit: CBI.13 - section rules -// from CBI.3.0 +// Rule unit: CBI.13 rule "CBI.13.1: Redacted because Section contains a vertebrate" when $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -295,7 +291,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate" }); end -//from CBI.3.2 rule "CBI.13.2: Do not redact because Section does not contain a vertebrate" when $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -304,8 +299,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate" .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found")); end - -// from CBI.4.0 rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section" when $section: Section(!hasTables(), @@ -323,8 +316,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction }); end - -// from CBI.5.0 rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section" when $section: Section(!hasTables(), @@ -346,7 +337,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica }); end -// From CBI.8.0 rule "CBI.13.5: Redacted because Section contains must_redact entity" when $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -362,6 +352,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity" }); end + // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when @@ -1133,7 +1124,7 @@ rule "AI.7.0: Add all NER Entities of type Address" end -//------------------------------------ Manual redaction rules ------------------------------------ +//------------------------------------ Manual changes rules ------------------------------------ // Rule unit: MAN.0 rule "MAN.0.0: Apply manual resize redaction" @@ -1323,7 +1314,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -1462,7 +1452,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index fa34d1fd..db704e92 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -397,7 +397,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -536,7 +535,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 840ce18e..a6cb0f97 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -489,7 +489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -639,7 +638,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 699c4182..553f97fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -389,31 +389,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" retract($entity) end - -rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" - salience 64 - when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) - then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end - - -// Rule unit: X.2 -rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" - salience 64 - when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) - then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end - - rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when @@ -563,7 +538,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity" retract($nonManualEntity); end - rule "X.11.2: Remove non manual entity which are equal to manual entity" salience 70 when diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java index 94975b8e..c002fed3 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java @@ -4,13 +4,11 @@ import java.io.File; import java.io.FileOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory; import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser; import com.knecon.fforesight.utility.rules.management.models.BasicRule; import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint; -import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier; import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; import lombok.SneakyThrows; @@ -23,7 +21,6 @@ import lombok.experimental.UtilityClass; @UtilityClass public class RuleFileMigrator { - @SneakyThrows public void migrateFile(File ruleFile) { @@ -53,21 +50,12 @@ public class RuleFileMigrator { private static void replaceRuleIdentifiers(RuleFileBluePrint combinedBluePrint, RuleFileBluePrint ruleFileBluePrint) { - Map identifierReplaceMap = Map.of("CBI.7.0", "CBI.16.0", "CBI.7.1", "CBI.16.1", "CBI.7.2", "CBI.16.2", "CBI.7.3", "CBI.16.3"); - for (String identifier : identifierReplaceMap.keySet()) { - RuleIdentifier ruleId = RuleIdentifier.fromString(identifier); - RuleIdentifier otherRuleId = RuleIdentifier.fromString(identifierReplaceMap.get(identifier)); - - List rulesToAdd = combinedBluePrint.findRuleByIdentifier(otherRuleId); - List otherRulesToAdd = combinedBluePrint.findRuleByIdentifier(ruleId); - boolean removeRules = ruleFileBluePrint.removeRule(ruleId); - boolean removeOtherRules = ruleFileBluePrint.removeRule(otherRuleId); - if (removeRules) { - rulesToAdd.forEach(ruleFileBluePrint::addRule); - } - if (removeOtherRules) { - otherRulesToAdd.forEach(ruleFileBluePrint::addRule); + for (BasicRule rule : combinedBluePrint.getAllRules()) { + if (ruleFileBluePrint.findRuleByIdentifier(rule.identifier()).isEmpty()) { + continue; } + ruleFileBluePrint.removeRule(rule.identifier()); + ruleFileBluePrint.addRule(rule); } } diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 7d841391..8b46eecc 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIde import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine; import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; @@ -55,8 +56,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 851e61cb..a78911b9 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; @@ -47,8 +48,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java index bea04152..98e9e680 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java @@ -5,7 +5,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator; @@ -25,17 +24,14 @@ public class RuleFileMigrationTest { // Put your redaction service drools paths and dossier-templates paths both RM and DM here static final List ruleFileDirs = List.of( - //"/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools", - // "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2" - "/Users/maverickstuder/Documents/PM" - - ); - + "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools", + "/home/kschuettler/iqser/redaction/dossier-templates-v2", + "/home/kschuettler/iqser/fforesight/dossier-templates-v2"); @Test @SneakyThrows - @Disabled +// @Disabled void migrateAllEntityRules() { for (String ruleFileDir : ruleFileDirs) {