Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'DM-357' into 'master'

Browse Source 
DM-357: apply rectangle, change annotationId of found entities to the id of the manualredaction

Closes DM-357

See merge request redactmanager/redaction-service!96
This commit is contained in:
Kilian Schüttler 2023-08-17 15:48:08 +02:00
commit bcda7e4c5c
8 changed files with 93 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/MatchedRuleHolder.java
View File

@ -42,15 +42,27 @@ public interface MatchedRuleHolder {
}
default void apply(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
default void redact(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
if (legalBasis.isBlank() || legalBasis.isEmpty()) {
throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
}
apply(ruleIdentifier, reason, legalBasis);
}
default void apply(@NonNull String ruleIdentifier, String reason, String legalBasis) {
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build());
}
default void apply(@NonNull String ruleIdentifier, String reason) {
apply(ruleIdentifier, reason, "n-a");
}
default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) {
addMatchedRule(MatchedRule.builder()

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Table.java
View File

@ -247,6 +247,18 @@ public class Table implements SemanticNode {
}
/**
* Streams all Headers and checks if any equal the provided string.
*
* @param header string to check the headers for
* @return true, if at least one header equals the provided string
*/
public boolean hasHeaderIgnoreCase(String header) {
return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock().getSearchText().strip().toLowerCase(Locale.ENGLISH).equals(header.toLowerCase(Locale.ENGLISH)));
}
/**
* Checks if this table has a column with the provided header and any of the table cells in that column contain the provided value.
*

46
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java
View File

@ -147,9 +147,6 @@ public class EntityCreationService {
}
public Stream<RedactionEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
@ -168,9 +165,6 @@ public class EntityCreationService {
}
public Stream<RedactionEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.getTextBlock();
@ -261,6 +255,21 @@ public class EntityCreationService {
}
public Stream<RedactionEntity> lineAfterStringsIgnoreCase(List<String> strings, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.getTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(strings, true);
return searchImplementation.getBoundaries(textBlock, node.getBoundary())
.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
}
public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.getTextBlock();
@ -274,6 +283,19 @@ public class EntityCreationService {
}
public Stream<RedactionEntity> lineAfterStringIgnoreCase(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.getTextBlock();
return RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, textBlock)
.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
}
public Stream<RedactionEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
return tableNode.streamTableCells()
@ -412,6 +434,16 @@ public class EntityCreationService {
}
public Stream<RedactionEntity> byStringIgnoreCase(String keyword, String type, EntityType entityType, SemanticNode node) {
return RedactionSearchUtility.findBoundariesByStringIgnoreCase(keyword, node.getTextBlock())
.stream()
.map(boundary -> byBoundary(boundary, type, entityType, node))
.filter(Optional::isPresent)
.map(Optional::get);
}
public Stream<RedactionEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) {
return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get);
@ -429,6 +461,7 @@ public class EntityCreationService {
.map(Optional::get);
}
public Optional<RedactionEntity> semanticNodeAfterString(String string, String type, EntityType entityType, SemanticNode node) {
if (!node.containsString(string)) {
@ -438,6 +471,7 @@ public class EntityCreationService {
return byBoundary(boundary, type, entityType, node);
}
public Optional<RedactionEntity> bySemanticNode(SemanticNode node, String type, EntityType entityType) {
Boundary boundary = node.getTextBlock().getBoundary();

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/CustomEntityCreationAdapter.java
View File

@ -4,6 +4,7 @@ import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.LinkedList;
@ -105,6 +106,15 @@ public class CustomEntityCreationAdapter {
}
correctEntity.setDictionaryEntry(entityIdentifier.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(entityIdentifier.isDossierDictionaryEntry());
// TODO: refactor this away! This is only needed so the persistence service can apply the correct comment and ManualChanges.
// It would be better, if the redaction-service returns a map of annotationId changes and the persistence service then migrates the annotationIds of Comments and ManualRedactions
List<RedactionPosition> redactionPositionsWithIdOfManualRedaction = new ArrayList<>(correctEntity.getRedactionPositionsPerPage().size());
for (RedactionPosition redactionPosition : correctEntity.getRedactionPositionsPerPage()) {
redactionPositionsWithIdOfManualRedaction.add(new RedactionPosition(entityIdentifier.getId(), redactionPosition.getPage(), redactionPosition.getRectanglePerLine()));
}
correctEntity.setRedactionPositionsPerPage(redactionPositionsWithIdOfManualRedaction);
}

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityIdentifier.java
View File

@ -16,6 +16,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class EntityIdentifier {
// must be used for comments to work correctly
String id;
String value;
List<RectangleWithPage> entityPosition;
String ruleIdentifier;
@ -27,13 +29,15 @@ public class EntityIdentifier {
boolean applied;
boolean isDictionaryEntry;
boolean isDossierDictionaryEntry;
boolean rectangle;
public static EntityIdentifier fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
return new EntityIdentifier(redactionLogEntry.getValue(),
return new EntityIdentifier(redactionLogEntry.getId(),
redactionLogEntry.getValue(),
rectangleWithPages,
ruleIdentifier,
redactionLogEntry.getReason(),
@ -43,26 +47,27 @@ public class EntityIdentifier {
redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY,
redactionLogEntry.isRedacted(),
redactionLogEntry.isDictionaryEntry(),
redactionLogEntry.isDossierDictionaryEntry());
redactionLogEntry.isDossierDictionaryEntry(),
redactionLogEntry.isRectangle());
}
public static EntityIdentifier fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) {
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
return new EntityIdentifier(manualRedactionEntry.getValue(),
return new EntityIdentifier(manualRedactionEntry.getAnnotationId(),
manualRedactionEntry.getValue(),
rectangleWithPages,
"MAN.0.0",
manualRedactionEntry.getReason(),
manualRedactionEntry.getLegalBasis(),
manualRedactionEntry.getType(), manualRedactionEntry.getSection(),
manualRedactionEntry.getType(),
manualRedactionEntry.getSection(),
EntityType.ENTITY,
true,
false,
false);
false,
manualRedactionEntry.isRectangle());
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -22,7 +22,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityIdentifier;
import com.iqser.red.service.redaction.v1.server.redaction.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -120,7 +119,7 @@ public class RedactionLogCreatorService {
List<Integer> pageNumbers = entityIdentifier.getEntityPosition().stream().map(RectangleWithPage::pageNumber).toList();
List<Rectangle2D> rectanglesPerLine = entityIdentifier.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList();
return RedactionLogEntry.builder()
.id(IdBuilder.buildId(pageNumbers, rectanglesPerLine, entityIdentifier.getType(), entityIdentifier.getEntityType().name()))
.id(entityIdentifier.getId())
.color(getColor(entityIdentifier.getType(), dossierTemplateId, entityIdentifier.isApplied()))
.reason(entityIdentifier.getReason())
.legalBasis(entityIdentifier.getLegalBasis())
@ -133,6 +132,7 @@ public class RedactionLogCreatorService {
.section(entityIdentifier.getSection())
.sectionNumber(0)
.matchedRule("ManualRedaction")
.rectangle(entityIdentifier.isRectangle())
.isDictionaryEntry(entityIdentifier.isDictionaryEntry())
.textAfter("")
.textBefore("")

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/RedactionEntityTest.java
View File

@ -56,7 +56,7 @@ public class RedactionEntityTest {
entity.skip("aaaaaaaaaa", "");
});
assertThrows(IllegalArgumentException.class, () -> {
entity.apply("CBI.0.0", "", "");
entity.redact("CBI.0.0", "", "");
});
entity.skip("CBI.2.0", "");
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.2.0");

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java
View File

@ -11,6 +11,7 @@ import java.util.stream.Stream;
import org.drools.io.ClassPathResource;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@ -74,6 +75,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest {
@Test
@Disabled // Enable if you fix the TODO in EntityCreationService
@SneakyThrows
public void testMigration() {