Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* fix rebased tests
This commit is contained in:
Kilian Schuettler 2023-05-02 18:07:31 +02:00
parent cdab1d6945
commit fdfb4c4071
8 changed files with 37 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java
View File

@ -29,8 +29,6 @@ public class TablePageBlock extends AbstractPageBlock {
private String headline;
private int unrotatedRowCount;
private int unrotatedColCount;
private int rowCount = -1;
private int colCount = -1;
private List<List<Cell>> rows;
@ -71,19 +69,13 @@ public class TablePageBlock extends AbstractPageBlock {
public int getRowCount() {
if (rowCount == -1) {
rowCount = getRows().size();
}
return rowCount;
return getRows().size();
}
public int getColCount() {
if (colCount == -1) {
colCount = getRows().stream().mapToInt(List::size).max().orElse(0);
}
return colCount;
return getRows().stream().mapToInt(List::size).max().orElse(0);
}
@ -268,7 +260,7 @@ public class TablePageBlock extends AbstractPageBlock {
prevX = x;
}
if (prevY != null && prevX != null) {
if (prevY != null && prevX != null && !row.isEmpty()) {
matrix.add(row);
}
prevY = y;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -167,6 +167,10 @@ public class EntityCreationService {
RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry));
mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDossierDictionaryEntry));
mergedEntity.setIgnored(entitiesToMerge.stream().allMatch(RedactionEntity::isIgnored));
mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved));
addEntityToGraph(mergedEntity, node);
return mergedEntity;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -113,11 +114,12 @@ public class RedactionLogCreatorService {
public RedactionLogEntry createRedactionLogEntry(ImageNode image, String dossierTemplateId) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase();
return RedactionLogEntry.builder()
.id(image.getId())
.color(getColor(image.getImageType().toString(), dossierTemplateId, image.isRedaction()))
.isImage(true)
.type(image.getImageType().toString().toLowerCase())
.type(imageType)
.redacted(image.isRedaction())
.reason(image.getRedactionReason())
.legalBasis(image.getLegalBasis())

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -131,7 +131,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2);
assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
}
@ -170,7 +170,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2);
assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
}

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java
View File

@ -239,7 +239,7 @@ public class PdfSegmentationServiceTest {
validateTableSize(document, 1);
validateTable(document, 0, 8, 9, 0, 2);
validateTable(document, 0, 8, 8, 0, 2);
List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
"Author, date",
@ -562,7 +562,7 @@ public class PdfSegmentationServiceTest {
private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
List<List<Cell>> rows = table.getRows();
int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
@ -576,7 +576,7 @@ public class PdfSegmentationServiceTest {
private void validateTable(Document document, int tableIndex, List<List<String>> values) {
Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
List<List<Cell>> rows = table.getRows();
List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl
View File

@ -1,8 +0,0 @@
import java.util.List
import com.iqser.red.service.redaction.v1.server.DroolsListBugTest
rule "bugtest"
when
then
DroolsListBugTest.callInFunction();
end

20
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -582,6 +582,16 @@ rule "Apply image recategorization"
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
then
$contained.removeFromGraph();
retract($contained);
end
rule "merge intersecting Entities of same type"
salience 64
when
@ -596,16 +606,6 @@ rule "merge intersecting Entities of same type"
insert(mergedEntity);
end
rule "remove Entity contained by Entity of same type"
salience 64
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
then
$contained.removeFromGraph();
retract($contained);
end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when

15
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -90,6 +90,16 @@ rule "Always redact PII"
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
then
$contained.removeFromGraph();
retract($contained);
end
rule "merge intersecting Entities of same type"
salience 64
when
@ -163,7 +173,6 @@ rule "run local dictionary search"
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
Set<RedactionEntity> redactionEntities = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
System.out.printf("local dictionary search found %d entities\n", redactionEntities.size());
redactionEntities.forEach(redactionEntity -> insert(redactionEntity));
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(redactionEntity -> insert(redactionEntity));
end