Merge branch 'reformat' into 'master'

reformat

See merge request redactmanager/redaction-service!303
This commit is contained in:
Kilian Schüttler 2024-03-01 15:49:37 +01:00
commit 7e144e30bf
127 changed files with 2363 additions and 1225 deletions

View File

@ -15,4 +15,5 @@ public class AnalyzeResponse {
private String fileId; private String fileId;
private List<UnprocessedManualEntity> unprocessedManualEntities; private List<UnprocessedManualEntity> unprocessedManualEntities;
} }

View File

@ -17,4 +17,5 @@ public class DroolsSyntaxDeprecatedWarnings {
Integer line; Integer line;
Integer column; Integer column;
String message; String message;
} }

View File

@ -17,4 +17,5 @@ public class DroolsSyntaxErrorMessage {
Integer line; Integer line;
Integer column; Integer column;
String message; String message;
} }

View File

@ -28,6 +28,7 @@ public class DroolsSyntaxValidation {
getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build()); getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build());
} }
public boolean isCompiled() { public boolean isCompiled() {
return droolsSyntaxErrorMessages.isEmpty(); return droolsSyntaxErrorMessages.isEmpty();

View File

@ -24,4 +24,5 @@ public class UnprocessedManualEntity {
private String section; private String section;
@Builder.Default @Builder.Default
private List<Position> positions = new ArrayList<>(); private List<Position> positions = new ArrayList<>();
} }

View File

@ -35,11 +35,14 @@ public class Application {
SpringApplication.run(Application.class, args); SpringApplication.run(Application.class, args);
} }
@Bean @Bean
public ObservedAspect observedAspect(ObservationRegistry observationRegistry) { public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
return new ObservedAspect(observationRegistry); return new ObservedAspect(observationRegistry);
} }
@Bean @Bean
public TimedAspect timedAspect(MeterRegistry registry) { public TimedAspect timedAspect(MeterRegistry registry) {

View File

@ -95,6 +95,7 @@ public class DeprecatedElementsFinder {
return this.deprecatedClasses; return this.deprecatedClasses;
} }
private String getMethodSignature(Method method) { private String getMethodSignature(Method method) {
String methodName = method.getName(); String methodName = method.getName();

View File

@ -16,7 +16,7 @@ public class RedisCachingConfiguration {
public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() { public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() {
return (builder) -> builder.withCacheConfiguration("documentDataCache", return (builder) -> builder.withCacheConfiguration("documentDataCache",
RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues()); RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues());
} }

View File

@ -1,6 +1,5 @@
package com.iqser.red.service.redaction.v1.server.client.model; package com.iqser.red.service.redaction.v1.server.client.model;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

View File

@ -13,7 +13,6 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
public class RuleBuilderController implements RuleBuilderResource { public class RuleBuilderController implements RuleBuilderResource {
@Override @Override
public RuleBuilderModel getRuleBuilderModel() { public RuleBuilderModel getRuleBuilderModel() {

View File

@ -92,11 +92,16 @@ public class LegacyRedactionLogMergeService {
return redactionLog; return redactionLog;
} }
public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) { public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count(); return createManualRedactionWrappers(manualRedactions).stream()
.map(ManualRedactionWrapper::getId)
.distinct()
.count();
} }
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) { private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>(); List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();

View File

@ -21,7 +21,9 @@ public class LegacyVersion0MigrationService {
public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) { public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
List<RedactionLogEntry> mergedEntries = new LinkedList<>(); List<RedactionLogEntry> mergedEntries = new LinkedList<>();
Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId)); Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry()
.stream()
.collect(Collectors.groupingBy(RedactionLogEntry::getId));
for (List<RedactionLogEntry> entries : entriesById.values()) { for (List<RedactionLogEntry> entries : entriesById.values()) {
if (entries.isEmpty()) { if (entries.isEmpty()) {
@ -33,7 +35,10 @@ public class LegacyVersion0MigrationService {
continue; continue;
} }
List<RedactionLogEntry> sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList(); List<RedactionLogEntry> sortedEntries = entries.stream()
.sorted(Comparator.comparing(entry -> entry.getChanges()
.get(0).getDateTime()))
.toList();
RedactionLogEntry initialEntry = sortedEntries.get(0); RedactionLogEntry initialEntry = sortedEntries.get(0);
for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) { for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {

View File

@ -14,4 +14,5 @@ public record KieWrapper(KieContainer container, long rulesVersion) {
return container != null && rulesVersion >= 0; return container != null && rulesVersion >= 0;
} }
} }

View File

@ -19,4 +19,5 @@ public class MigratedEntityLog {
MigratedIds migratedIds; MigratedIds migratedIds;
EntityLog entityLog; EntityLog entityLog;
} }

View File

@ -27,13 +27,15 @@ public class NerEntities {
public boolean hasEntitiesOfType(String type) { public boolean hasEntitiesOfType(String type) {
return nerEntityList.stream().anyMatch(nerEntity -> nerEntity.type.equals(type)); return nerEntityList.stream()
.anyMatch(nerEntity -> nerEntity.type.equals(type));
} }
public Stream<NerEntity> streamEntitiesOfType(String type) { public Stream<NerEntity> streamEntitiesOfType(String type) {
return nerEntityList.stream().filter(nerEntity -> nerEntity.type().equals(type)); return nerEntityList.stream()
.filter(nerEntity -> nerEntity.type().equals(type));
} }

View File

@ -88,7 +88,8 @@ public class Entity {
.textAfter(e.getTextAfter()) .textAfter(e.getTextAfter())
.startOffset(e.getStartOffset()) .startOffset(e.getStartOffset())
.endOffset(e.getEndOffset()) .endOffset(e.getEndOffset())
.length(Optional.ofNullable(e.getValue()).orElse("").length()) .length(Optional.ofNullable(e.getValue())
.orElse("").length())
.imageHasTransparency(e.isImageHasTransparency()) .imageHasTransparency(e.isImageHasTransparency())
.isDictionaryEntry(e.isDictionaryEntry()) .isDictionaryEntry(e.isDictionaryEntry())
.isDossierDictionaryEntry(e.isDossierDictionaryEntry()) .isDossierDictionaryEntry(e.isDossierDictionaryEntry())

View File

@ -53,7 +53,8 @@ public class Dictionary {
public boolean hasLocalEntries() { public boolean hasLocalEntries() {
return dictionaryModels.stream().anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty()); return dictionaryModels.stream()
.anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty());
} }
@ -116,12 +117,18 @@ public class Dictionary {
} }
localAccessMap.get(type) localAccessMap.get(type)
.getLocalEntriesWithMatchedRules() .getLocalEntriesWithMatchedRules()
.merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet())); .merge(cleanedValue.trim(),
matchedRulesSet,
(set1, set2) -> Stream.concat(set1.stream(), set2.stream())
.collect(Collectors.toSet()));
if (alsoAddLastname) { if (alsoAddLastname) {
String lastname = cleanedValue.split(" ")[0]; String lastname = cleanedValue.split(" ")[0];
localAccessMap.get(type) localAccessMap.get(type)
.getLocalEntriesWithMatchedRules() .getLocalEntriesWithMatchedRules()
.merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet())); .merge(lastname,
matchedRulesSet,
(set1, set2) -> Stream.concat(set1.stream(), set2.stream())
.collect(Collectors.toSet()));
} }
} }
@ -153,7 +160,10 @@ public class Dictionary {
} else { } else {
splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n")); splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n"));
} }
return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList(); return splitAuthorNames.stream()
.map(String::trim)
.filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches())
.toList();
} }
} }

View File

@ -57,14 +57,18 @@ public class DictionaryModel implements Serializable {
this.falsePositives = falsePositives; this.falsePositives = falsePositives;
this.falseRecommendations = falseRecommendations; this.falseRecommendations = falseRecommendations;
this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()), this.entriesSearch = new SearchImplementation(this.entries.stream()
caseInsensitive); .filter(e -> !e.isDeleted())
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()), .map(DictionaryEntryModel::getValue)
caseInsensitive); .collect(Collectors.toList()), caseInsensitive);
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntryModel::getValue)
.collect(Collectors.toList()), caseInsensitive);
this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
.filter(e -> !e.isDeleted()) .filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue) .map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive); .collect(Collectors.toList()), caseInsensitive);
} }
@ -81,8 +85,10 @@ public class DictionaryModel implements Serializable {
public SearchImplementation getEntriesSearch() { public SearchImplementation getEntriesSearch() {
if (entriesSearch == null) { if (entriesSearch == null) {
this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), this.entriesSearch = new SearchImplementation(this.entries.stream()
caseInsensitive); .filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive);
} }
return entriesSearch; return entriesSearch;
} }
@ -92,9 +98,9 @@ public class DictionaryModel implements Serializable {
if (falsePositiveSearch == null) { if (falsePositiveSearch == null) {
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream() this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
.filter(e -> !e.isDeleted()) .filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue) .map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive); .collect(Collectors.toList()), caseInsensitive);
} }
return falsePositiveSearch; return falsePositiveSearch;
} }
@ -104,14 +110,16 @@ public class DictionaryModel implements Serializable {
if (falseRecommendationsSearch == null) { if (falseRecommendationsSearch == null) {
this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
.filter(e -> !e.isDeleted()) .filter(e -> !e.isDeleted())
.map(DictionaryEntry::getValue) .map(DictionaryEntry::getValue)
.collect(Collectors.toList()), caseInsensitive); .collect(Collectors.toList()), caseInsensitive);
} }
return falseRecommendationsSearch; return falseRecommendationsSearch;
} }
public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) { public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value; var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
return localEntriesWithMatchedRules.get(cleanedValue); return localEntriesWithMatchedRules.get(cleanedValue);

View File

@ -76,7 +76,9 @@ public class SearchImplementation {
if (ignoreCase) { if (ignoreCase) {
textToCheck = textToCheck.toLowerCase(Locale.ROOT); textToCheck = textToCheck.toLowerCase(Locale.ROOT);
} }
return this.pattern.matcher(textToCheck).results().findAny().isPresent(); return this.pattern.matcher(textToCheck).results()
.findAny()
.isPresent();
} else { } else {
return this.trie.containsMatch(textToCheck); return this.trie.containsMatch(textToCheck);
} }
@ -89,9 +91,14 @@ public class SearchImplementation {
return new ArrayList<>(); return new ArrayList<>();
} }
if (this.pattern != null) { if (this.pattern != null) {
return this.pattern.matcher(text).results().map(r -> new TextRange(r.start(), r.end())).collect(Collectors.toList()); return this.pattern.matcher(text).results()
.map(r -> new TextRange(r.start(), r.end()))
.collect(Collectors.toList());
} else { } else {
return this.trie.parseText(text).stream().map(r -> new TextRange(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); return this.trie.parseText(text)
.stream()
.map(r -> new TextRange(r.getStart(), r.getEnd() + 1))
.collect(Collectors.toList());
} }
} }
@ -103,9 +110,14 @@ public class SearchImplementation {
} }
CharSequence subSequence = text.subSequence(region.start(), region.end()); CharSequence subSequence = text.subSequence(region.start(), region.end());
if (this.pattern != null) { if (this.pattern != null) {
return this.pattern.matcher(subSequence).results().map(r -> new TextRange(r.start() + region.start(), r.end() + region.start())).collect(Collectors.toList()); return this.pattern.matcher(subSequence).results()
.map(r -> new TextRange(r.start() + region.start(), r.end() + region.start()))
.collect(Collectors.toList());
} else { } else {
return this.trie.parseText(subSequence).stream().map(r -> new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1)).collect(Collectors.toList()); return this.trie.parseText(subSequence)
.stream()
.map(r -> new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1))
.collect(Collectors.toList());
} }
} }
@ -120,9 +132,14 @@ public class SearchImplementation {
if (ignoreCase) { if (ignoreCase) {
textToCheck = textToCheck.toLowerCase(Locale.ROOT); textToCheck = textToCheck.toLowerCase(Locale.ROOT);
} }
return this.pattern.matcher(textToCheck).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList()); return this.pattern.matcher(textToCheck).results()
.map(r -> new MatchPosition(r.start(), r.end()))
.collect(Collectors.toList());
} else { } else {
return this.trie.parseText(textToCheck).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); return this.trie.parseText(textToCheck)
.stream()
.map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1))
.collect(Collectors.toList());
} }
} }

View File

@ -40,7 +40,10 @@ public class DocumentTree {
public TextBlock buildTextBlock() { public TextBlock buildTextBlock() {
return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); return allEntriesInOrder().map(Entry::getNode)
.filter(SemanticNode::isLeaf)
.map(SemanticNode::getLeafTextBlock)
.collect(new TextBlockCollector());
} }
@ -114,13 +117,16 @@ public class DocumentTree {
public Stream<SemanticNode> childNodes(List<Integer> treeId) { public Stream<SemanticNode> childNodes(List<Integer> treeId) {
return getEntryById(treeId).children.stream().map(Entry::getNode); return getEntryById(treeId).children.stream()
.map(Entry::getNode);
} }
public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) { public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) {
return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode); return getEntryById(treeId).children.stream()
.filter(entry -> entry.node.getType().equals(nodeType))
.map(Entry::getNode);
} }
@ -199,26 +205,32 @@ public class DocumentTree {
public Stream<Entry> allEntriesInOrder() { public Stream<Entry> allEntriesInOrder() {
return Stream.of(root).flatMap(DocumentTree::flatten); return Stream.of(root)
.flatMap(DocumentTree::flatten);
} }
public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) { public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) {
return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten); return getEntryById(parentId).children.stream()
.flatMap(DocumentTree::flatten);
} }
@Override @Override
public String toString() { public String toString() {
return String.join("\n", allEntriesInOrder().map(Entry::toString).toList()); return String.join("\n",
allEntriesInOrder().map(Entry::toString)
.toList());
} }
private static Stream<Entry> flatten(Entry entry) { private static Stream<Entry> flatten(Entry entry) {
return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTree::flatten)); return Stream.concat(Stream.of(entry),
entry.children.stream()
.flatMap(DocumentTree::flatten));
} }

View File

@ -92,8 +92,13 @@ public class TextRange implements Comparable<TextRange> {
public List<TextRange> split(List<Integer> splitIndices) { public List<TextRange> split(List<Integer> splitIndices) {
if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) { if (splitIndices.stream()
throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this)); .anyMatch(idx -> !this.contains(idx))) {
throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
splitIndices.stream()
.filter(idx -> !this.contains(idx))
.toList(),
this));
} }
List<TextRange> splitBoundaries = new LinkedList<>(); List<TextRange> splitBoundaries = new LinkedList<>();
int previousIndex = start; int previousIndex = start;
@ -113,8 +118,14 @@ public class TextRange implements Comparable<TextRange> {
public static TextRange merge(Collection<TextRange> boundaries) { public static TextRange merge(Collection<TextRange> boundaries) {
int minStart = boundaries.stream().mapToInt(TextRange::start).min().orElseThrow(IllegalArgumentException::new); int minStart = boundaries.stream()
int maxEnd = boundaries.stream().mapToInt(TextRange::end).max().orElseThrow(IllegalArgumentException::new); .mapToInt(TextRange::start)
.min()
.orElseThrow(IllegalArgumentException::new);
int maxEnd = boundaries.stream()
.mapToInt(TextRange::end)
.max()
.orElseThrow(IllegalArgumentException::new);
return new TextRange(minStart, maxEnd); return new TextRange(minStart, maxEnd);
} }

View File

@ -35,14 +35,16 @@ public interface IEntity {
default String value() { default String value() {
return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue()); return getManualOverwrite().getValue()
.orElse(getValue() == null ? "" : getValue());
} }
// Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner. // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
default boolean applied() { default boolean applied() {
return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied()); return getManualOverwrite().getApplied()
.orElse(getMatchedRule().isApplied());
} }
@ -54,19 +56,22 @@ public interface IEntity {
default boolean ignored() { default boolean ignored() {
return getManualOverwrite().getIgnored().orElse(getMatchedRule().isIgnored()); return getManualOverwrite().getIgnored()
.orElse(getMatchedRule().isIgnored());
} }
default boolean removed() { default boolean removed() {
return getManualOverwrite().getRemoved().orElse(getMatchedRule().isRemoved()); return getManualOverwrite().getRemoved()
.orElse(getMatchedRule().isRemoved());
} }
default boolean resized() { default boolean resized() {
return getManualOverwrite().getResized().orElse(false); return getManualOverwrite().getResized()
.orElse(false);
} }
@ -133,12 +138,12 @@ public interface IEntity {
throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
} }
getMatchedRuleList().add(MatchedRule.builder() getMatchedRuleList().add(MatchedRule.builder()
.ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
.reason(reason) .reason(reason)
.legalBasis(legalBasis) .legalBasis(legalBasis)
.applied(true) .applied(true)
.writeValueWithLineBreaks(true) .writeValueWithLineBreaks(true)
.build()); .build());
} }
@ -148,12 +153,12 @@ public interface IEntity {
throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
} }
getMatchedRuleList().add(MatchedRule.builder() getMatchedRuleList().add(MatchedRule.builder()
.ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
.reason(reason) .reason(reason)
.legalBasis(legalBasis) .legalBasis(legalBasis)
.applied(true) .applied(true)
.references(new HashSet<>(references)) .references(new HashSet<>(references))
.build()); .build());
} }
@ -207,7 +212,8 @@ public interface IEntity {
default String legalBasis() { default String legalBasis() {
return getManualOverwrite().getLegalBasis().orElse(getMatchedRule().getLegalBasis()); return getManualOverwrite().getLegalBasis()
.orElse(getMatchedRule().getLegalBasis());
} }
} }

View File

@ -52,7 +52,8 @@ public final class MatchedRule implements Comparable<MatchedRule> {
if (!this.isApplied()) { if (!this.isApplied()) {
return this; return this;
} }
return MatchedRule.builder().ruleIdentifier(getRuleIdentifier()) return MatchedRule.builder()
.ruleIdentifier(getRuleIdentifier())
.writeValueWithLineBreaks(this.isWriteValueWithLineBreaks()) .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
.legalBasis(this.getLegalBasis()) .legalBasis(this.getLegalBasis())
.reason(this.getReason()) .reason(this.getReason())
@ -97,7 +98,19 @@ public final class MatchedRule implements Comparable<MatchedRule> {
@Override @Override
public String toString() { public String toString() {
return "MatchedRule[ruleIdentifier=" + ruleIdentifier + ", reason=" + reason + ", legalBasis=" + legalBasis + ", applied=" + applied + ", writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", references=" + references + ']'; return "MatchedRule[ruleIdentifier="
+ ruleIdentifier
+ ", reason="
+ reason
+ ", legalBasis="
+ legalBasis
+ ", applied="
+ applied
+ ", writeValueWithLineBreaks="
+ writeValueWithLineBreaks
+ ", references="
+ references
+ ']';
} }
} }

View File

@ -67,7 +67,13 @@ public class TextEntity implements IEntity {
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build(); return TextEntity.builder()
.id(buildId(node, textRange, type, entityType))
.type(type)
.entityType(entityType)
.textRange(textRange)
.manualOverwrite(new ManualChangeOverwrite(entityType))
.build();
} }
@ -80,7 +86,13 @@ public class TextEntity implements IEntity {
private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) { private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange); Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name()); return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(),
rectanglesPerLinePerPage.values()
.stream()
.flatMap(Collection::stream)
.toList(),
type,
entityType.name());
} }
@ -89,15 +101,18 @@ public class TextEntity implements IEntity {
duplicateTextRanges.add(textRange); duplicateTextRanges.add(textRange);
} }
public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) { public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
return intersectingNodes.stream().anyMatch(clazz::isInstance); return intersectingNodes.stream()
.anyMatch(clazz::isInstance);
} }
public boolean occursInNode(SemanticNode semanticNode) { public boolean occursInNode(SemanticNode semanticNode) {
return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode)); return intersectingNodes.stream()
.anyMatch(node -> node.equals(semanticNode));
} }
@ -146,7 +161,10 @@ public class TextEntity implements IEntity {
.min(Comparator.comparingInt(Page::getNumber)) .min(Comparator.comparingInt(Page::getNumber))
.orElseThrow(() -> new RuntimeException("No Positions found on any page!")); .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList(); positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet()
.stream()
.map(entry -> buildPositionOnPage(firstPage, id, entry))
.toList();
} }
return positionsOnPagePerPage; return positionsOnPagePerPage;
} }
@ -194,7 +212,8 @@ public class TextEntity implements IEntity {
public boolean matchesAnnotationId(String manualRedactionId) { public boolean matchesAnnotationId(String manualRedactionId) {
return getPositionsOnPagePerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId)); return getPositionsOnPagePerPage().stream()
.anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
} }
@ -224,14 +243,16 @@ public class TextEntity implements IEntity {
@Override @Override
public String type() { public String type() {
return getManualOverwrite().getType().orElse(type); return getManualOverwrite().getType()
.orElse(type);
} }
@Override @Override
public String value() { public String value() {
return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value); return getManualOverwrite().getValue()
.orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
} }
} }

View File

@ -65,13 +65,15 @@ public class Document implements GenericSemanticNode {
public List<Section> getMainSections() { public List<Section> getMainSections() {
return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList()); return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
.collect(Collectors.toList());
} }
public Stream<TextBlock> streamTerminalTextBlocksInOrder() { public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock); return streamAllNodes().filter(SemanticNode::isLeaf)
.map(SemanticNode::getLeafTextBlock);
} }
@ -92,13 +94,16 @@ public class Document implements GenericSemanticNode {
@Override @Override
public Headline getHeadline() { public Headline getHeadline() {
return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseGet(Headline::empty); return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node)
.findFirst()
.orElseGet(Headline::empty);
} }
private Stream<SemanticNode> streamAllNodes() { private Stream<SemanticNode> streamAllNodes() {
return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode); return documentTree.allEntriesInOrder()
.map(DocumentTree.Entry::getNode);
} }

View File

@ -106,7 +106,9 @@ public class Headline implements GenericSemanticNode {
public boolean hasParagraphs() { public boolean hasParagraphs() {
return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent(); return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH)
.findFirst()
.isPresent();
} }
} }

View File

@ -136,7 +136,7 @@ public class Image implements GenericSemanticNode, IEntity {
Map<Page, Rectangle2D> bboxImage = image.getBBox(); Map<Page, Rectangle2D> bboxImage = image.getBBox();
Map<Page, Rectangle2D> bbox = this.getBBox(); Map<Page, Rectangle2D> bbox = this.getBBox();
//image needs to be on the same page //image needs to be on the same page
if(bboxImage.get(this.page) != null) { if (bboxImage.get(this.page) != null) {
Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page)); Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
double calculatedIntersection = intersection.getWidth() * intersection.getHeight(); double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight(); double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();

View File

@ -45,7 +45,10 @@ public class Page {
public TextBlock getMainBodyTextBlock() { public TextBlock getMainBodyTextBlock() {
return mainBody.stream().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); return mainBody.stream()
.filter(SemanticNode::isLeaf)
.map(SemanticNode::getLeafTextBlock)
.collect(new TextBlockCollector());
} }
@ -54,4 +57,5 @@ public class Page {
return String.valueOf(number); return String.valueOf(number);
} }
} }

View File

@ -53,7 +53,8 @@ public class Section implements GenericSemanticNode {
public boolean hasTables() { public boolean hasTables() {
return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent(); return streamAllSubNodesOfType(NodeType.TABLE).findAny()
.isPresent();
} }
@ -68,7 +69,9 @@ public class Section implements GenericSemanticNode {
public TextBlock getTextBlock() { public TextBlock getTextBlock() {
if (textBlock == null) { if (textBlock == null) {
textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
.map(SemanticNode::getLeafTextBlock)
.collect(new TextBlockCollector());
} }
return textBlock; return textBlock;
} }

View File

@ -72,7 +72,11 @@ public class SectionIdentifier {
} }
identifiers.add(Integer.parseInt(numericalIdentifier.trim())); identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
} }
return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false); return new SectionIdentifier(Format.NUMERICAL,
identifierString,
identifiers.stream()
.toList(),
false);
} }

View File

@ -71,7 +71,10 @@ public interface SemanticNode {
*/ */
default Page getFirstPage() { default Page getFirstPage() {
return getTextBlock().getPages().stream().min(Comparator.comparingInt(Page::getNumber)).orElseThrow(); return getTextBlock().getPages()
.stream()
.min(Comparator.comparingInt(Page::getNumber))
.orElseThrow();
} }
@ -97,7 +100,8 @@ public interface SemanticNode {
*/ */
default boolean onPage(int pageNumber) { default boolean onPage(int pageNumber) {
return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber); return getPages().stream()
.anyMatch(page -> page.getNumber() == pageNumber);
} }
@ -249,7 +253,9 @@ public interface SemanticNode {
*/ */
default boolean hasEntitiesOfType(String type) { default boolean hasEntitiesOfType(String type) {
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type)); return getEntities().stream()
.filter(TextEntity::active)
.anyMatch(redactionEntity -> redactionEntity.type().equals(type));
} }
@ -262,7 +268,10 @@ public interface SemanticNode {
*/ */
default boolean hasEntitiesOfAnyType(String... types) { default boolean hasEntitiesOfAnyType(String... types) {
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type))); return getEntities().stream()
.filter(TextEntity::active)
.anyMatch(redactionEntity -> Arrays.stream(types)
.anyMatch(type -> redactionEntity.type().equals(type)));
} }
@ -275,7 +284,12 @@ public interface SemanticNode {
*/ */
default boolean hasEntitiesOfAllTypes(String... types) { default boolean hasEntitiesOfAllTypes(String... types) {
return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList()); return getEntities().stream()
.filter(TextEntity::active)
.map(TextEntity::type)
.collect(Collectors.toUnmodifiableSet())
.containsAll(Arrays.stream(types)
.toList());
} }
@ -288,7 +302,10 @@ public interface SemanticNode {
*/ */
default List<TextEntity> getEntitiesOfType(String type) { default List<TextEntity> getEntitiesOfType(String type) {
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList(); return getEntities().stream()
.filter(TextEntity::active)
.filter(redactionEntity -> redactionEntity.type().equals(type))
.toList();
} }
@ -301,7 +318,10 @@ public interface SemanticNode {
*/ */
default List<TextEntity> getEntitiesOfType(List<String> types) { default List<TextEntity> getEntitiesOfType(List<String> types) {
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList(); return getEntities().stream()
.filter(TextEntity::active)
.filter(redactionEntity -> redactionEntity.isAnyType(types))
.toList();
} }
@ -314,7 +334,11 @@ public interface SemanticNode {
*/ */
default List<TextEntity> getEntitiesOfType(String... types) { default List<TextEntity> getEntitiesOfType(String... types) {
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList(); return getEntities().stream()
.filter(TextEntity::active)
.filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types)
.toList()))
.toList();
} }
@ -328,7 +352,8 @@ public interface SemanticNode {
TextBlock textBlock = getTextBlock(); TextBlock textBlock = getTextBlock();
if (!textBlock.getAtomicTextBlocks().isEmpty()) { if (!textBlock.getAtomicTextBlocks().isEmpty()) {
return getTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage(); return getTextBlock().getAtomicTextBlocks()
.get(0).getNumberOnPage();
} else { } else {
return -1; return -1;
} }
@ -357,14 +382,16 @@ public interface SemanticNode {
return getTextBlock().getSearchText().contains(string); return getTextBlock().getSearchText().contains(string);
} }
Set<LayoutEngine> getEngines(); Set<LayoutEngine> getEngines();
default void addEngine(LayoutEngine engine) { default void addEngine(LayoutEngine engine) {
getEngines().add(engine); getEngines().add(engine);
} }
/** /**
* Checks whether this SemanticNode contains all the provided Strings. * Checks whether this SemanticNode contains all the provided Strings.
* *
@ -373,7 +400,8 @@ public interface SemanticNode {
*/ */
default boolean containsAllStrings(String... strings) { default boolean containsAllStrings(String... strings) {
return Arrays.stream(strings).allMatch(this::containsString); return Arrays.stream(strings)
.allMatch(this::containsString);
} }
@ -385,7 +413,8 @@ public interface SemanticNode {
*/ */
default boolean containsAnyString(String... strings) { default boolean containsAnyString(String... strings) {
return Arrays.stream(strings).anyMatch(this::containsString); return Arrays.stream(strings)
.anyMatch(this::containsString);
} }
@ -397,7 +426,8 @@ public interface SemanticNode {
*/ */
default boolean containsAnyString(List<String> strings) { default boolean containsAnyString(List<String> strings) {
return strings.stream().anyMatch(this::containsString); return strings.stream()
.anyMatch(this::containsString);
} }
@ -421,7 +451,8 @@ public interface SemanticNode {
*/ */
default boolean containsAnyStringIgnoreCase(String... strings) { default boolean containsAnyStringIgnoreCase(String... strings) {
return Arrays.stream(strings).anyMatch(this::containsStringIgnoreCase); return Arrays.stream(strings)
.anyMatch(this::containsStringIgnoreCase);
} }
@ -433,7 +464,8 @@ public interface SemanticNode {
*/ */
default boolean containsAllStringsIgnoreCase(String... strings) { default boolean containsAllStringsIgnoreCase(String... strings) {
return Arrays.stream(strings).allMatch(this::containsStringIgnoreCase); return Arrays.stream(strings)
.allMatch(this::containsStringIgnoreCase);
} }
@ -445,7 +477,9 @@ public interface SemanticNode {
*/ */
default boolean containsWord(String word) { default boolean containsWord(String word) {
return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word)); return getTextBlock().getWords()
.stream()
.anyMatch(s -> s.equals(word));
} }
@ -457,7 +491,10 @@ public interface SemanticNode {
*/ */
default boolean containsWordIgnoreCase(String word) { default boolean containsWordIgnoreCase(String word) {
return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH))); return getTextBlock().getWords()
.stream()
.map(String::toLowerCase)
.anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
} }
@ -469,7 +506,10 @@ public interface SemanticNode {
*/ */
default boolean containsAnyWord(String... words) { default boolean containsAnyWord(String... words) {
return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals)); return Arrays.stream(words)
.anyMatch(word -> getTextBlock().getWords()
.stream()
.anyMatch(word::equals));
} }
@ -481,7 +521,12 @@ public interface SemanticNode {
*/ */
default boolean containsAnyWordIgnoreCase(String... words) { default boolean containsAnyWordIgnoreCase(String... words) {
return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals)); return Arrays.stream(words)
.map(String::toLowerCase)
.anyMatch(word -> getTextBlock().getWords()
.stream()
.map(String::toLowerCase)
.anyMatch(word::equals));
} }
@ -493,7 +538,10 @@ public interface SemanticNode {
*/ */
default boolean containsAllWords(String... words) { default boolean containsAllWords(String... words) {
return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals)); return Arrays.stream(words)
.allMatch(word -> getTextBlock().getWords()
.stream()
.anyMatch(word::equals));
} }
@ -505,7 +553,12 @@ public interface SemanticNode {
*/ */
default boolean containsAllWordsIgnoreCase(String... words) { default boolean containsAllWordsIgnoreCase(String... words) {
return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals)); return Arrays.stream(words)
.map(String::toLowerCase)
.allMatch(word -> getTextBlock().getWords()
.stream()
.map(String::toLowerCase)
.anyMatch(word::equals));
} }
@ -545,7 +598,11 @@ public interface SemanticNode {
*/ */
default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) { default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
return getBBox().entrySet().stream().filter(entry -> entry.getKey().getNumber() == pageNumber).map(Map.Entry::getValue).anyMatch(rect -> rect.intersects(x, y, w, h)); return getBBox().entrySet()
.stream()
.filter(entry -> entry.getKey().getNumber() == pageNumber)
.map(Map.Entry::getValue)
.anyMatch(rect -> rect.intersects(x, y, w, h));
} }
@ -598,7 +655,8 @@ public interface SemanticNode {
*/ */
default Stream<SemanticNode> streamAllSubNodes() { default Stream<SemanticNode> streamAllSubNodes() {
return getDocumentTree().allSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode); return getDocumentTree().allSubEntriesInOrder(getTreeId())
.map(DocumentTree.Entry::getNode);
} }
@ -609,7 +667,9 @@ public interface SemanticNode {
*/ */
default Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType) { default Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType) {
return getDocumentTree().allSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode); return getDocumentTree().allSubEntriesInOrder(getTreeId())
.filter(entry -> entry.getType().equals(nodeType))
.map(DocumentTree.Entry::getNode);
} }
@ -648,7 +708,8 @@ public interface SemanticNode {
if (isLeaf()) { if (isLeaf()) {
return getTextBlock().getPositionsPerPage(textRange); return getTextBlock().getPositionsPerPage(textRange);
} }
Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst(); Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange))
.findFirst();
if (containingChildNode.isEmpty()) { if (containingChildNode.isEmpty()) {
return getTextBlock().getPositionsPerPage(textRange); return getTextBlock().getPositionsPerPage(textRange);
} }
@ -698,8 +759,12 @@ public interface SemanticNode {
private Map<Page, Rectangle2D> getBBoxFromChildren() { private Map<Page, Rectangle2D> getBBoxFromChildren() {
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>(); Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList(); List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox)
Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet()); .toList();
Set<Page> pages = childrenBBoxes.stream()
.flatMap(map -> map.keySet()
.stream())
.collect(Collectors.toSet());
for (Page page : pages) { for (Page page : pages) {
Rectangle2D bBoxOnPage = childrenBBoxes.stream() Rectangle2D bBoxOnPage = childrenBBoxes.stream()
.filter(childBboxPerPage -> childBboxPerPage.containsKey(page)) .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
@ -717,7 +782,9 @@ public interface SemanticNode {
private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() { private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>(); Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage)); Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks()
.stream()
.collect(Collectors.groupingBy(AtomicTextBlock::getPage));
atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks))); atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
return bBoxPerPage; return bBoxPerPage;
} }

View File

@ -79,7 +79,9 @@ public class TableCell implements GenericSemanticNode {
} }
if (textBlock == null) { if (textBlock == null) {
textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
.map(SemanticNode::getLeafTextBlock)
.collect(new TextBlockCollector());
} }
return textBlock; return textBlock;
} }

View File

@ -61,6 +61,7 @@ public class AtomicTextBlock implements TextBlock {
return lineBreaks.size() + 1; return lineBreaks.size() + 1;
} }
public static AtomicTextBlock empty(Long textBlockIdx, int stringOffset, Page page, int numberOnPage, SemanticNode parent) { public static AtomicTextBlock empty(Long textBlockIdx, int stringOffset, Page page, int numberOnPage, SemanticNode parent) {
return AtomicTextBlock.builder() return AtomicTextBlock.builder()
@ -77,10 +78,7 @@ public class AtomicTextBlock implements TextBlock {
} }
public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData atomicTextBlockData, public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData atomicTextBlockData, DocumentPositionData atomicPositionBlockData, SemanticNode parent, Page page) {
DocumentPositionData atomicPositionBlockData,
SemanticNode parent,
Page page) {
return AtomicTextBlock.builder() return AtomicTextBlock.builder()
.id(atomicTextBlockData.getId()) .id(atomicTextBlockData.getId())
@ -88,8 +86,10 @@ public class AtomicTextBlock implements TextBlock {
.page(page) .page(page)
.textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd())) .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
.searchText(atomicTextBlockData.getSearchText()) .searchText(atomicTextBlockData.getSearchText())
.lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList()) .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed()
.stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList()) .toList())
.stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed()
.toList())
.positions(toRectangle2DList(atomicPositionBlockData.getPositions())) .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
.parent(parent) .parent(parent)
.build(); .build();
@ -98,7 +98,9 @@ public class AtomicTextBlock implements TextBlock {
private static List<Rectangle2D> toRectangle2DList(float[][] positions) { private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList(); return Arrays.stream(positions)
.map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
.toList();
} }
@ -118,6 +120,7 @@ public class AtomicTextBlock implements TextBlock {
return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start()); return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
} }
public List<String> getWords() { public List<String> getWords() {
if (words == null) { if (words == null) {
@ -144,9 +147,9 @@ public class AtomicTextBlock implements TextBlock {
public int getNextLinebreak(int fromIndex) { public int getNextLinebreak(int fromIndex) {
return lineBreaks.stream()// return lineBreaks.stream()//
.filter(linebreak -> linebreak > fromIndex - textRange.start()) // .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
.findFirst() // .findFirst() //
.orElse(searchText.length()) + textRange.start(); .orElse(searchText.length()) + textRange.start();
} }
@ -154,9 +157,9 @@ public class AtomicTextBlock implements TextBlock {
public int getPreviousLinebreak(int fromIndex) { public int getPreviousLinebreak(int fromIndex) {
return lineBreaks.stream()// return lineBreaks.stream()//
.filter(linebreak -> linebreak <= fromIndex - textRange.start())// .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
.reduce((a, b) -> b)// .reduce((a, b) -> b)//
.orElse(0) + textRange.start(); .orElse(0) + textRange.start();
} }
@ -209,7 +212,10 @@ public class AtomicTextBlock implements TextBlock {
return ""; return "";
} }
Set<Integer> lbInBoundary = lineBreaks.stream().map(i -> i + textRange.start()).filter(textRange::contains).collect(Collectors.toSet()); Set<Integer> lbInBoundary = lineBreaks.stream()
.map(i -> i + textRange.start())
.filter(textRange::contains)
.collect(Collectors.toSet());
if (textRange.end() == getTextRange().end()) { if (textRange.end() == getTextRange().end()) {
lbInBoundary.add(getTextRange().end()); lbInBoundary.add(getTextRange().end());
} }
@ -235,7 +241,10 @@ public class AtomicTextBlock implements TextBlock {
private List<Integer> getAllLineBreaksInBoundary(TextRange textRange) { private List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList(); return getLineBreaks().stream()
.map(linebreak -> linebreak + this.textRange.start())
.filter(textRange::contains)
.toList();
} }

View File

@ -44,7 +44,8 @@ public class ConcatenatedTextBlock implements TextBlock {
this.atomicTextBlocks.add(firstTextBlock); this.atomicTextBlocks.add(firstTextBlock);
textRange = new TextRange(firstTextBlock.getTextRange().start(), firstTextBlock.getTextRange().end()); textRange = new TextRange(firstTextBlock.getTextRange().start(), firstTextBlock.getTextRange().end());
atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat); atomicTextBlocks.subList(1, atomicTextBlocks.size())
.forEach(this::concat);
} }
@ -65,7 +66,10 @@ public class ConcatenatedTextBlock implements TextBlock {
private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) { private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
return atomicTextBlocks.stream().filter(textBlock -> textBlock.getTextRange().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new); return atomicTextBlocks.stream()
.filter(textBlock -> textBlock.getTextRange().contains(stringIdx))
.findAny()
.orElseThrow(IndexOutOfBoundsException::new);
} }
@ -99,14 +103,18 @@ public class ConcatenatedTextBlock implements TextBlock {
@Override @Override
public List<String> getWords() { public List<String> getWords() {
return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList(); return atomicTextBlocks.stream()
.map(AtomicTextBlock::getWords)
.flatMap(Collection::stream)
.toList();
} }
@Override @Override
public int numberOfLines() { public int numberOfLines() {
return atomicTextBlocks.stream().mapToInt(AtomicTextBlock::numberOfLines).sum(); return atomicTextBlocks.stream()
.mapToInt(AtomicTextBlock::numberOfLines).sum();
} }
@ -127,7 +135,10 @@ public class ConcatenatedTextBlock implements TextBlock {
@Override @Override
public List<Integer> getLineBreaks() { public List<Integer> getLineBreaks() {
return getAtomicTextBlocks().stream().flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks().stream()).toList(); return getAtomicTextBlocks().stream()
.flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks()
.stream())
.toList();
} }
@ -202,7 +213,8 @@ public class ConcatenatedTextBlock implements TextBlock {
AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1); AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage,
lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end()))); lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(),
stringTextRange.end())));
return rectanglesPerLinePerPage; return rectanglesPerLinePerPage;
} }
@ -239,7 +251,10 @@ public class ConcatenatedTextBlock implements TextBlock {
private Map<Page, List<Rectangle2D>> mergeEntityPositionsWithSamePageNode(Map<Page, List<Rectangle2D>> map1, Map<Page, List<Rectangle2D>> map2) { private Map<Page, List<Rectangle2D>> mergeEntityPositionsWithSamePageNode(Map<Page, List<Rectangle2D>> map1, Map<Page, List<Rectangle2D>> map2) {
Map<Page, List<Rectangle2D>> mergedMap = new HashMap<>(map1); Map<Page, List<Rectangle2D>> mergedMap = new HashMap<>(map1);
map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (l1, l2) -> Stream.concat(l1.stream(), l2.stream()).toList())); map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode,
rectangles,
(l1, l2) -> Stream.concat(l1.stream(), l2.stream())
.toList()));
return mergedMap; return mergedMap;
} }

View File

@ -18,8 +18,10 @@ public interface TextBlock extends CharSequence {
String getSearchText(); String getSearchText();
List<String> getWords(); List<String> getWords();
List<AtomicTextBlock> getAtomicTextBlocks(); List<AtomicTextBlock> getAtomicTextBlocks();
@ -35,7 +37,6 @@ public interface TextBlock extends CharSequence {
TextRange getLineTextRange(int lineNumber); TextRange getLineTextRange(int lineNumber);
List<Integer> getLineBreaks(); List<Integer> getLineBreaks();
@ -71,6 +72,7 @@ public interface TextBlock extends CharSequence {
return RectangleTransformations.rectangle2DBBox(getLinePositions(lineNumber)); return RectangleTransformations.rectangle2DBBox(getLinePositions(lineNumber));
} }
default String searchTextWithLineBreaks() { default String searchTextWithLineBreaks() {
return subSequenceWithLineBreaks(getTextRange()); return subSequenceWithLineBreaks(getTextRange());
@ -85,7 +87,9 @@ public interface TextBlock extends CharSequence {
default Set<Page> getPages() { default Set<Page> getPages() {
return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet()); return getAtomicTextBlocks().stream()
.map(AtomicTextBlock::getPage)
.collect(Collectors.toUnmodifiableSet());
} }

View File

@ -9,7 +9,8 @@ public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) { public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
return ruleUnits.stream() return ruleUnits.stream()
.filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst(); .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit))
.findFirst();
} }
} }

View File

@ -37,13 +37,17 @@ public final class RuleFileBluePrint {
public Optional<RuleClass> findRuleClassByType(RuleType ruleType) { public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst(); return ruleClasses.stream()
.filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType))
.findFirst();
} }
public Set<String> getImportSplitByKeyword() { public Set<String> getImportSplitByKeyword() {
return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet()); return Arrays.stream(imports.replaceAll("\n", "").split("import"))
.map(String::trim)
.collect(Collectors.toSet());
} }
@ -53,11 +57,15 @@ public final class RuleFileBluePrint {
return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits) return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits)
.orElse(Collections.emptyList()) .orElse(Collections.emptyList())
.stream() .stream()
.flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) .flatMap(ruleUnit -> ruleUnit.rules()
.stream()
.filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
.toList(); .toList();
} }
return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit())) return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()))
.map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) .map(ruleUnit -> ruleUnit.rules()
.stream()
.filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
.orElse(Stream.empty()) .orElse(Stream.empty())
.toList(); .toList();
} }
@ -65,13 +73,18 @@ public final class RuleFileBluePrint {
public List<RuleIdentifier> getAllRuleIdentifiers() { public List<RuleIdentifier> getAllRuleIdentifiers() {
return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList()); return streamAllRules().map(BasicRule::getIdentifier)
.collect(Collectors.toList());
} }
public Stream<BasicRule> streamAllRules() { public Stream<BasicRule> streamAllRules() {
return getRuleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream); return getRuleClasses().stream()
.map(RuleClass::ruleUnits)
.flatMap(Collection::stream)
.map(RuleUnit::rules)
.flatMap(Collection::stream);
} }

View File

@ -42,8 +42,8 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
public boolean matches(RuleIdentifier ruleIdentifier) { public boolean matches(RuleIdentifier ruleIdentifier) {
return ruleIdentifier.type().equals(this.type()) && // return ruleIdentifier.type().equals(this.type()) && //
(Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && //
(Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id()));
} }

View File

@ -70,6 +70,7 @@ public class MessagingConfiguration {
.build(); .build();
} }
@Bean @Bean
public Queue redactionAnalysisResponseQueue() { public Queue redactionAnalysisResponseQueue() {

View File

@ -51,14 +51,14 @@ public class RedactionMessageReceiver {
// This prevents from endless retries oom errors. // This prevents from endless retries oom errors.
if (message.getMessageProperties().isRedelivered()) { if (message.getMessageProperties().isRedelivered()) {
var errorMessage = format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", var errorMessage = format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
analyzeRequest.getDossierId(), analyzeRequest.getDossierId(),
analyzeRequest.getFileId()); analyzeRequest.getFileId());
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
new FileErrorInfo(errorMessage, new FileErrorInfo(errorMessage,
priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE,
"redaction-service", "redaction-service",
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
throw new AmqpRejectAndDontRequeueException(errorMessage); throw new AmqpRejectAndDontRequeueException(errorMessage);
} }
@ -84,9 +84,9 @@ public class RedactionMessageReceiver {
log.debug(analyzeRequest.getManualRedactions().toString()); log.debug(analyzeRequest.getManualRedactions().toString());
result = analyzeService.analyze(analyzeRequest); result = analyzeService.analyze(analyzeRequest);
log.info("Successfully analyzed dossier {} file {} took: {} s", log.info("Successfully analyzed dossier {} file {} took: {} s",
analyzeRequest.getDossierId(), analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
format("%.2f", result.getDuration() / 1000.0)); format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------------------------"); log.info("----------------------------------------------------------------------------------");
break; break;
@ -96,9 +96,9 @@ public class RedactionMessageReceiver {
log.debug(analyzeRequest.getManualRedactions().toString()); log.debug(analyzeRequest.getManualRedactions().toString());
result = analyzeService.reanalyze(analyzeRequest); result = analyzeService.reanalyze(analyzeRequest);
log.info("Successfully reanalyzed dossier {} file {} took: {} s", log.info("Successfully reanalyzed dossier {} file {} took: {} s",
analyzeRequest.getDossierId(), analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
format("%.2f", result.getDuration() / 1000.0)); format("%.2f", result.getDuration() / 1000.0));
log.info("----------------------------------------------------------------------------------"); log.info("----------------------------------------------------------------------------------");
break; break;
case SURROUNDING_TEXT_ANALYSIS: case SURROUNDING_TEXT_ANALYSIS:
@ -106,9 +106,7 @@ public class RedactionMessageReceiver {
log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.debug(analyzeRequest.getManualRedactions().toString()); log.debug(analyzeRequest.getManualRedactions().toString());
unprocessedChangesService.analyseSurroundingText(analyzeRequest); unprocessedChangesService.analyseSurroundingText(analyzeRequest);
log.info("Successful Surrounding Text Analysis dossier {} file {} ", log.info("Successful Surrounding Text Analysis dossier {} file {} ", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
analyzeRequest.getDossierId(),
analyzeRequest.getFileId());
log.info("-------------------------------------------------------------------------------------------------"); log.info("-------------------------------------------------------------------------------------------------");
shouldRespond = false; shouldRespond = false;
break; break;
@ -137,8 +135,8 @@ public class RedactionMessageReceiver {
log.warn("Failed to process analyze request: {}", analyzeRequest, e); log.warn("Failed to process analyze request: {}", analyzeRequest, e);
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
} }
@ -153,8 +151,8 @@ public class RedactionMessageReceiver {
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp); log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp);
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp)); new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp));
} }
} }

View File

@ -23,13 +23,15 @@ public class ComponentLogCreatorService {
public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) { public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
Map<String, List<ComponentLogEntryValue>> map = new HashMap<>(); Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
components.stream().sorted(ComponentComparator.first()).forEach(component -> { components.stream()
ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component); .sorted(ComponentComparator.first())
map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue); .forEach(component -> {
}); ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
List<ComponentLogEntry> componentLogComponents = map map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
.entrySet() });
.stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue())) List<ComponentLogEntry> componentLogComponents = map.entrySet()
.stream()
.map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
.toList(); .toList();
return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents); return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
} }
@ -38,24 +40,36 @@ public class ComponentLogCreatorService {
private ComponentLogEntryValue buildComponentLogEntry(Component component) { private ComponentLogEntryValue buildComponentLogEntry(Component component) {
return ComponentLogEntryValue.builder() return ComponentLogEntryValue.builder()
.value(component.getValue()).originalValue(component.getValue()) .value(component.getValue())
.originalValue(component.getValue())
.componentRuleId(component.getMatchedRule().toString()) .componentRuleId(component.getMatchedRule().toString())
.valueDescription(component.getValueDescription()) .valueDescription(component.getValueDescription())
.componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.first()).toList())) .componentLogEntityReferences(toComponentEntityReferences(component.getReferences()
.stream()
.sorted(EntityComparators.first())
.toList()))
.build(); .build();
} }
private List<ComponentLogEntityReference> toComponentEntityReferences(List<Entity> references) { private List<ComponentLogEntityReference> toComponentEntityReferences(List<Entity> references) {
return references.stream().map(this::toComponentEntityReference).toList(); return references.stream()
.map(this::toComponentEntityReference)
.toList();
} }
private ComponentLogEntityReference toComponentEntityReference(Entity entity) { private ComponentLogEntityReference toComponentEntityReference(Entity entity) {
return ComponentLogEntityReference.builder().id(entity.getId()) return ComponentLogEntityReference.builder()
.page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule()) .id(entity.getId())
.page(entity.getPositions()
.stream()
.findFirst()
.map(Position::getPageNumber)
.orElse(0))
.entityRuleId(entity.getMatchedRule())
.type(entity.getType()) .type(entity.getType())
.build(); .build();
} }

View File

@ -56,10 +56,11 @@ public class DictionarySearchService {
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream() .stream()
.filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> { .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY))
entity.setDictionaryEntry(true); .ifPresent(entity -> {
entity.setDossierDictionaryEntry(isDossierDictionaryEntry); entity.setDictionaryEntry(true);
})); entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
}));
} }
} }

View File

@ -106,41 +106,47 @@ public class DictionaryService {
List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary(); List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> { dictionaryModel.getEntries()
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { .forEach(dictionaryEntry -> {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
} newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}); }
dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> { });
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { dictionaryModel.getFalsePositives()
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); .forEach(dictionaryEntry -> {
} if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
}); newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> { }
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { });
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); dictionaryModel.getFalseRecommendations()
} .forEach(dictionaryEntry -> {
}); if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
}); });
if (dossierDictionaryExists(dossierId)) { if (dossierDictionaryExists(dossierId)) {
dictionaryModels = getDossierDictionary(dossierId).getDictionary(); dictionaryModels = getDossierDictionary(dossierId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> { dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> { dictionaryModel.getEntries()
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { .forEach(dictionaryEntry -> {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
} newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}); }
dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> { });
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { dictionaryModel.getFalsePositives()
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); .forEach(dictionaryEntry -> {
} if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
}); newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> { }
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { });
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); dictionaryModel.getFalseRecommendations()
} .forEach(dictionaryEntry -> {
}); if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
}
});
}); });
} }
@ -155,84 +161,120 @@ public class DictionaryService {
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId, var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId,
true); true);
if (CollectionUtils.isNotEmpty(typeResponse)) { if (CollectionUtils.isNotEmpty(typeResponse)) {
List<DictionaryModel> dictionary = typeResponse.stream().map(t -> { List<DictionaryModel> dictionary = typeResponse.stream()
.map(t -> {
Optional<DictionaryModel> optionalOldModel; Optional<DictionaryModel> optionalOldModel;
if (dossierId == null) { if (dossierId == null) {
var representation = getDossierTemplateDictionary(dossierTemplateId); var representation = getDossierTemplateDictionary(dossierTemplateId);
optionalOldModel = representation != null ? representation.getDictionary() optionalOldModel = representation != null ? representation.getDictionary()
.stream() .stream()
.filter(f -> f.getType().equals(t.getType())) .filter(f -> f.getType().equals(t.getType()))
.findAny() : Optional.empty(); .findAny() : Optional.empty();
} else { } else {
var representation = getDossierDictionary(dossierId); var representation = getDossierDictionary(dossierId);
optionalOldModel = representation != null ? representation.getDictionary() optionalOldModel = representation != null ? representation.getDictionary()
.stream() .stream()
.filter(f -> f.getType().equals(t.getType())) .filter(f -> f.getType().equals(t.getType()))
.findAny() : Optional.empty(); .findAny() : Optional.empty();
} }
Set<DictionaryEntryModel> entries = new HashSet<>(); Set<DictionaryEntryModel> entries = new HashSet<>();
Set<DictionaryEntryModel> falsePositives = new HashSet<>(); Set<DictionaryEntryModel> falsePositives = new HashSet<>();
Set<DictionaryEntryModel> falseRecommendations = new HashSet<>(); Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();
DictionaryEntries newEntries = getEntries(t.getId(), currentVersion); DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
var newValues = newEntries.getEntries().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet()); var newValues = newEntries.getEntries()
var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
optionalOldModel.ifPresent(oldDictionaryModel -> {
});
if (optionalOldModel.isPresent()) {
var oldModel = optionalOldModel.get();
if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) {
// add old entries from existing DictionaryModel but exclude lower case representation
entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())).toList());
falsePositives.addAll(oldModel.getFalsePositives()
.stream() .stream()
.filter(f -> !newFalsePositivesValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())) .map(DictionaryEntry::getValue)
.toList()); .collect(Collectors.toSet());
falseRecommendations.addAll(oldModel.getFalseRecommendations() var newFalsePositivesValues = newEntries.getFalsePositives()
.stream() .stream()
.filter(f -> !newFalseRecommendationsValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())) .map(DictionaryEntry::getValue)
.toList()); .collect(Collectors.toSet());
} else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) { var newFalseRecommendationsValues = newEntries.getFalseRecommendations()
// add old entries from existing DictionaryModel but exclude upper case representation
entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList());
falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream() .stream()
.filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT))) .map(DictionaryEntry::getValue)
.toList()); .collect(Collectors.toSet());
} else { optionalOldModel.ifPresent(oldDictionaryModel -> {
// add old entries from existing DictionaryModel
entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList());
falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue())).toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations().stream().filter(f -> !newFalseRecommendationsValues.contains(f.getValue())).toList());
}
}
// Add Increments });
entries.addAll(newEntries.getEntries()); if (optionalOldModel.isPresent()) {
falsePositives.addAll(newEntries.getFalsePositives()); var oldModel = optionalOldModel.get();
falseRecommendations.addAll(newEntries.getFalseRecommendations()); if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) {
// add old entries from existing DictionaryModel but exclude lower case representation
entries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newValues.stream()
.map(s -> s.toLowerCase(Locale.ROOT))
.toList().contains(f.getValue()))
.toList());
falsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newFalsePositivesValues.stream()
.map(s -> s.toLowerCase(Locale.ROOT))
.toList().contains(f.getValue()))
.toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newFalseRecommendationsValues.stream()
.map(s -> s.toLowerCase(Locale.ROOT))
.toList().contains(f.getValue()))
.toList());
} else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) {
// add old entries from existing DictionaryModel but exclude upper case representation
entries.addAll(oldModel.getEntries()
.stream()
.filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
.toList());
falsePositives.addAll(oldModel.getFalsePositives()
.stream()
.filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
.toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
.toList());
return new DictionaryModel(t.getType(), } else {
t.getRank(), // add old entries from existing DictionaryModel
convertColor(t.getHexColor()), entries.addAll(oldModel.getEntries()
t.isCaseInsensitive(), .stream()
t.isHint(), .filter(f -> !newValues.contains(f.getValue()))
entries, .toList());
falsePositives, falsePositives.addAll(oldModel.getFalsePositives()
falseRecommendations, .stream()
dossierId != null); .filter(f -> !newFalsePositivesValues.contains(f.getValue()))
}).sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()); .toList());
falseRecommendations.addAll(oldModel.getFalseRecommendations()
.stream()
.filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
.toList());
}
}
// Add Increments
entries.addAll(newEntries.getEntries());
falsePositives.addAll(newEntries.getFalsePositives());
falseRecommendations.addAll(newEntries.getFalseRecommendations());
return new DictionaryModel(t.getType(),
t.getRank(),
convertColor(t.getHexColor()),
t.isCaseInsensitive(),
t.isHint(),
entries,
falsePositives,
falseRecommendations,
dossierId != null);
})
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList());
dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm)); dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
@ -264,17 +306,17 @@ public class DictionaryService {
var type = dictionaryClient.getDictionaryForType(typeId, fromVersion); var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);
Set<DictionaryEntryModel> entries = type.getEntries() != null ? new HashSet<>(type.getEntries() Set<DictionaryEntryModel> entries = type.getEntries() != null ? new HashSet<>(type.getEntries()
.stream() .stream()
.map(DictionaryEntryModel::new) .map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>(); .collect(Collectors.toSet())) : new HashSet<>();
Set<DictionaryEntryModel> falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries() Set<DictionaryEntryModel> falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries()
.stream() .stream()
.map(DictionaryEntryModel::new) .map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>(); .collect(Collectors.toSet())) : new HashSet<>();
Set<DictionaryEntryModel> falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries() Set<DictionaryEntryModel> falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries()
.stream() .stream()
.map(DictionaryEntryModel::new) .map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>(); .collect(Collectors.toSet())) : new HashSet<>();
if (type.isCaseInsensitive()) { if (type.isCaseInsensitive()) {
entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
@ -282,10 +324,10 @@ public class DictionaryService {
falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
} }
log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}", log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}",
entries.size(), entries.size(),
falsePositives.size(), falsePositives.size(),
falseRecommendations.size(), falseRecommendations.size(),
typeId); typeId);
return new DictionaryEntries(entries, falsePositives, falseRecommendations); return new DictionaryEntries(entries, falsePositives, falseRecommendations);
} }
@ -300,7 +342,8 @@ public class DictionaryService {
@SneakyThrows @SneakyThrows
public float[] getColor(String type, String dossierTemplateId) { public float[] getColor(String type, String dossierTemplateId) {
DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type); DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap()
.get(type);
if (model != null) { if (model != null) {
return model.getColor(); return model.getColor();
} }
@ -311,7 +354,8 @@ public class DictionaryService {
@SneakyThrows @SneakyThrows
public boolean isHint(String type, String dossierTemplateId) { public boolean isHint(String type, String dossierTemplateId) {
DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type); DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap()
.get(type);
if (model != null) { if (model != null) {
return model.isHint(); return model.isHint();
} }
@ -335,15 +379,20 @@ public class DictionaryService {
var dossierRepresentation = getDossierDictionary(dossierId); var dossierRepresentation = getDossierDictionary(dossierId);
var dossierDictionaries = dossierRepresentation.getDictionary(); var dossierDictionaries = dossierRepresentation.getDictionary();
mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries),
convertDictionaryModel(dossierDictionaries))); convertDictionaryModel(dossierDictionaries)));
dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
} else { } else {
mergedDictionaries = new ArrayList<>(); mergedDictionaries = new ArrayList<>();
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
} }
return new Dictionary(mergedDictionaries.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), return new Dictionary(mergedDictionaries.stream()
DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()),
DictionaryVersion.builder()
.dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
.dossierVersion(dossierDictionaryVersion)
.build());
} }
@ -371,14 +420,16 @@ public class DictionaryService {
@SneakyThrows @SneakyThrows
private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) { private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId); return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate()
.get(dossierTemplateId);
} }
@SneakyThrows @SneakyThrows
private DictionaryRepresentation getDossierDictionary(String dossierId) { private DictionaryRepresentation getDossierDictionary(String dossierId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId); return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier()
.get(dossierId);
} }
@ -421,14 +472,14 @@ public class DictionaryService {
return commonsDictionaries.stream() return commonsDictionaries.stream()
.map(cd -> new DictionaryModel(cd.getType(), .map(cd -> new DictionaryModel(cd.getType(),
cd.getRank(), cd.getRank(),
cd.getColor(), cd.getColor(),
cd.isCaseInsensitive(), cd.isCaseInsensitive(),
cd.isHint(), cd.isHint(),
cd.getEntries(), cd.getEntries(),
cd.getFalsePositives(), cd.getFalsePositives(),
cd.getFalseRecommendations(), cd.getFalseRecommendations(),
cd.isDossierDictionary())) cd.isDossierDictionary()))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }

View File

@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j;
public class EntityChangeLogService { public class EntityChangeLogService {
@Timed("redactmanager_computeChanges") @Timed("redactmanager_computeChanges")
public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, ManualRedactions manualRedactions, int analysisNumber) { public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
var now = OffsetDateTime.now(); var now = OffsetDateTime.now();
if (previousEntityLogEntries.isEmpty()) { if (previousEntityLogEntries.isEmpty()) {
@ -58,16 +58,12 @@ public class EntityChangeLogService {
entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now)); entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
} }
} }
addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now); addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
return hasChanges; return hasChanges;
} }
private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries, private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber, OffsetDateTime now) {
List<EntityLogEntry> newEntityLogEntries,
ManualRedactions manualRedactions,
int analysisNumber,
OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream() Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId) .map(EntityLogEntry::getId)

View File

@ -70,7 +70,7 @@ public class EntityLogCreatorService {
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber()); entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
return new EntityLog(redactionServiceSettings.getAnalysisVersion(), return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(), analyzeRequest.getAnalysisNumber(),
@ -128,10 +128,8 @@ public class EntityLogCreatorService {
.collect(Collectors.toList()); .collect(Collectors.toList());
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
newEntityLogEntries,
analyzeRequest.getManualRedactions(),
analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);

View File

@ -74,9 +74,9 @@ public class ManualChangesApplicationService {
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions() positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions()
.stream() .stream()
.map(ManualChangesApplicationService::toRectangle2D) .map(ManualChangesApplicationService::toRectangle2D)
.collect(Collectors.toList())); .collect(Collectors.toList()));
entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction); entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
@ -90,11 +90,17 @@ public class ManualChangesApplicationService {
if (closestEntity.isPresent()) { if (closestEntity.isPresent()) {
copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get()); copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get());
possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); possibleEntities.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
return; return;
} }
possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); possibleEntities.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
if (node.hasParent()) { if (node.hasParent()) {
node = node.getParent(); node = node.getParent();
@ -110,14 +116,18 @@ public class ManualChangesApplicationService {
Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes()); Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes());
Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes()); Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes());
Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized)); Sets.difference(currentIntersectingNodes, newIntersectingNodes)
Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized)); .forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized));
Sets.difference(newIntersectingNodes, currentIntersectingNodes)
.forEach(addedNode -> addedNode.getEntities().add(entityToBeResized));
Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages()); Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages());
Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages()); Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages());
Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized)); Sets.difference(currentIntersectingPages, newIntersectingPages)
Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized)); .forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized));
Sets.difference(newIntersectingPages, currentIntersectingPages)
.forEach(addedPage -> addedPage.getEntities().add(entityToBeResized));
entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes)); entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes));
@ -135,7 +145,10 @@ public class ManualChangesApplicationService {
if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) { if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {
return; return;
} }
var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList()); var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions()
.stream()
.map(ManualChangesApplicationService::toRectangle2D)
.toList());
image.setPosition(bBox); image.setPosition(bBox);
image.getManualOverwrite().addChange(manualResizeRedaction); image.getManualOverwrite().addChange(manualResizeRedaction);
} }

View File

@ -53,10 +53,13 @@ public class NotFoundImportedEntitiesService {
if (!notFoundEntities.isEmpty()) { if (!notFoundEntities.isEmpty()) {
// imported redactions present, intersections must be added with merged imported redactions // imported redactions present, intersections must be added with merged imported redactions
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities); Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities);
entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> { entityLog.getEntityLogEntry()
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); .stream()
addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber); .filter(entry -> !entry.getEngines().contains(Engine.IMPORTED))
}); .forEach(redactionLogEntry -> {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
});
} }
} }
@ -70,7 +73,10 @@ public class NotFoundImportedEntitiesService {
.map(RectangleWithPage::pageNumber) .map(RectangleWithPage::pageNumber)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber, pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber,
importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList()))); importedEntities.stream()
.filter(i -> pageNumber == i.getEntityPosition()
.get(0).pageNumber())
.collect(Collectors.toList())));
return importedRedactionsMap; return importedRedactionsMap;
} }

View File

@ -15,8 +15,12 @@ public class ComponentComparator implements Comparator<Component> {
@Override @Override
public int compare(Component component1, Component component2) { public int compare(Component component1, Component component2) {
var firstEntity1 = component1.getReferences().stream().min(EntityComparators.first()); var firstEntity1 = component1.getReferences()
var firstEntity2 = component2.getReferences().stream().min(EntityComparators.first()); .stream()
.min(EntityComparators.first());
var firstEntity2 = component2.getReferences()
.stream()
.min(EntityComparators.first());
if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) { if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) {
return 0; return 0;
} }

View File

@ -40,7 +40,8 @@ public class ComponentCreationService {
private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) { private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); var entitiesBySection = entities.stream()
.collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
Optional<SemanticNode> longestSection = entitiesBySection.entrySet() Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
.stream() .stream()
.sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()) .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed())
@ -79,14 +80,20 @@ public class ComponentCreationService {
public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) { public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback); String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback);
String value = entities.stream().min(EntityComparators.first()).map(Entity::getValue).orElse(fallback); String value = entities.stream()
.min(EntityComparators.first())
.map(Entity::getValue)
.orElse(fallback);
create(ruleIdentifier, name, value, valueDescription, entities); create(ruleIdentifier, name, value, valueDescription, entities);
} }
private static String joinTypes(Collection<Entity> entities) { private static String joinTypes(Collection<Entity> entities) {
return entities.stream().map(Entity::getType).distinct().collect(Collectors.joining(", ")); return entities.stream()
.map(Entity::getType)
.distinct()
.collect(Collectors.joining(", "));
} }
@ -104,12 +111,12 @@ public class ComponentCreationService {
referencedEntities.addAll(references); referencedEntities.addAll(references);
kieSession.insert(Component.builder() kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier)) .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.name(name) .name(name)
.value(value) .value(value)
.valueDescription(valueDescription) .valueDescription(valueDescription)
.references(new LinkedList<>(references)) .references(new LinkedList<>(references))
.build()); .build());
} }
@ -142,8 +149,11 @@ public class ComponentCreationService {
private static List<Entity> findEntitiesFromFirstSection(Collection<Entity> entities) { private static List<Entity> findEntitiesFromFirstSection(Collection<Entity> entities) {
var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); var entitiesBySection = entities.stream()
Optional<SemanticNode> firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first()); .collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
Optional<SemanticNode> firstSection = entitiesBySection.keySet()
.stream()
.min(SemanticNodeComparators.first());
if (firstSection.isEmpty()) { if (firstSection.isEmpty()) {
return Collections.emptyList(); return Collections.emptyList();
} }
@ -188,7 +198,10 @@ public class ComponentCreationService {
public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) { public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter); String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter);
String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(delimiter)); String value = entities.stream()
.sorted(EntityComparators.first())
.map(Entity::getValue)
.collect(Collectors.joining(delimiter));
create(ruleIdentifier, name, value, valueDescription, entities); create(ruleIdentifier, name, value, valueDescription, entities);
} }
@ -231,14 +244,20 @@ public class ComponentCreationService {
public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) { public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter); String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter);
String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter)); String value = entities.stream()
.sorted(EntityComparators.first())
.map(Entity::getValue)
.distinct()
.collect(Collectors.joining(delimiter));
create(ruleIdentifier, name, value, valueDescription, entities); create(ruleIdentifier, name, value, valueDescription, entities);
} }
private static int getTotalLengthOfEntities(Map.Entry<SemanticNode, List<Entity>> entry) { private static int getTotalLengthOfEntities(Map.Entry<SemanticNode, List<Entity>> entry) {
return entry.getValue().stream().mapToInt(Entity::getLength).sum(); return entry.getValue()
.stream()
.mapToInt(Entity::getLength).sum();
} }
@ -293,7 +312,10 @@ public class ComponentCreationService {
*/ */
public void uniqueValueCount(String ruleIdentifier, String name, Collection<Entity> entities) { public void uniqueValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
long count = entities.stream().map(Entity::getValue).distinct().count(); long count = entities.stream()
.map(Entity::getValue)
.distinct()
.count();
create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities); create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities);
} }
@ -307,18 +329,20 @@ public class ComponentCreationService {
*/ */
public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) { public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> { entities.stream()
.collect(Collectors.groupingBy(this::getFirstTable))
.forEach((optionalTable, groupedEntities) -> {
if (optionalTable.isEmpty()) { if (optionalTable.isEmpty()) {
return; return;
} }
long count = groupedEntities.stream() long count = groupedEntities.stream()
.collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1))) .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow)
.size(); .orElse(-1))).size();
create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities); create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
}); });
} }
@ -334,18 +358,20 @@ public class ComponentCreationService {
if (entities.isEmpty()) { if (entities.isEmpty()) {
return; return;
} }
entities.stream().sorted(EntityComparators.first()).forEach(entity -> { entities.stream()
BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH); .sorted(EntityComparators.first())
iterator.setText(entity.getValue()); .forEach(entity -> {
int start = iterator.first(); BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) { iterator.setText(entity.getValue());
create(ruleIdentifier, int start = iterator.first();
name, for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), create(ruleIdentifier,
String.format("Values of type '%s' as sentences", entity.getType()), name,
entity); entity.getValue().substring(start, end).replaceAll("\\n", "").trim(),
} String.format("Values of type '%s' as sentences", entity.getType()),
}); entity);
}
});
} }
@ -366,12 +392,12 @@ public class ComponentCreationService {
List<Entity> referenceList = new LinkedList<>(); List<Entity> referenceList = new LinkedList<>();
referenceList.add(reference); referenceList.add(reference);
kieSession.insert(Component.builder() kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier)) .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.name(name) .name(name)
.value(value) .value(value)
.valueDescription(valueDescription) .valueDescription(valueDescription)
.references(referenceList) .references(referenceList)
.build()); .build());
} }
@ -428,8 +454,10 @@ public class ComponentCreationService {
} }
String formattedDateStrings = Stream.concat(// String formattedDateStrings = Stream.concat(//
dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), // dates.stream()
unparsedDates.stream())// .sorted()
.map(date -> DateConverter.convertDate(date, resultFormat)), //
unparsedDates.stream())//
.collect(Collectors.joining(", ")); .collect(Collectors.joining(", "));
create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities); create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities);
@ -445,26 +473,34 @@ public class ComponentCreationService {
*/ */
public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) { public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) {
String types = entities.stream().map(Entity::getType).sorted(Comparator.reverseOrder()).distinct().collect(Collectors.joining(", ")); String types = entities.stream()
.map(Entity::getType)
.sorted(Comparator.reverseOrder())
.distinct()
.collect(Collectors.joining(", "));
String valueDescription = String.format("Combine values of %s that are in same table row", types); String valueDescription = String.format("Combine values of %s that are in same table row", types);
entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> { entities.stream()
if (optionalTable.isEmpty()) { .collect(Collectors.groupingBy(this::getFirstTable))
groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity)); .forEach((optionalTable, groupedEntities) -> {
} if (optionalTable.isEmpty()) {
groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
}
groupedEntities.stream() groupedEntities.stream()
.filter(entity -> entity.getContainingNode() instanceof TableCell) .filter(entity -> entity.getContainingNode() instanceof TableCell)
.collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())) .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())).entrySet()
.entrySet() .stream()
.stream() .sorted(Comparator.comparingInt(Map.Entry::getKey))
.sorted(Comparator.comparingInt(Map.Entry::getKey)) .map(Map.Entry::getValue)
.map(Map.Entry::getValue) .forEach(entitiesInSameRow -> create(ruleIdentifier,
.forEach(entitiesInSameRow -> create(ruleIdentifier, name,
name, entitiesInSameRow.stream()
entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")), .sorted(EntityComparators.first())
valueDescription, .map(Entity::getValue)
entitiesInSameRow)); .collect(Collectors.joining(", ")),
}); valueDescription,
entitiesInSameRow));
});
} }
@ -521,12 +557,12 @@ public class ComponentCreationService {
public void create(String ruleIdentifier, String name, String value) { public void create(String ruleIdentifier, String name, String value) {
kieSession.insert(Component.builder() kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier)) .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.name(name) .name(name)
.value(value) .value(value)
.valueDescription("") .valueDescription("")
.references(Collections.emptyList()) .references(Collections.emptyList())
.build()); .build());
} }
} }

View File

@ -40,7 +40,9 @@ public class DocumentGraphMapper {
DocumentTree documentTree = new DocumentTree(document); DocumentTree documentTree = new DocumentTree(document);
Context context = new Context(documentData, documentTree); Context context = new Context(documentData, documentTree);
context.pageData.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList()); context.pageData.addAll(Arrays.stream(documentData.getDocumentPages())
.map(DocumentGraphMapper::buildPage)
.toList());
context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context)); context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
@ -58,7 +60,9 @@ public class DocumentGraphMapper {
List<DocumentTree.Entry> newEntries = new LinkedList<>(); List<DocumentTree.Entry> newEntries = new LinkedList<>();
for (DocumentStructure.EntryData entryData : entries) { for (DocumentStructure.EntryData entryData : entries) {
List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList(); List<Page> pages = Arrays.stream(entryData.getPageNumbers())
.map(pageNumber -> getPage(pageNumber, context))
.toList();
SemanticNode node = switch (entryData.getType()) { SemanticNode node = switch (entryData.getType()) {
case SECTION -> buildSection(context); case SECTION -> buildSection(context);
@ -76,8 +80,10 @@ public class DocumentGraphMapper {
TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node); TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
node.setLeafTextBlock(textBlock); node.setLeafTextBlock(textBlock);
} }
List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList(); List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
entryData.getEngines().forEach(engine -> node.addEngine(engine)); .toList();
entryData.getEngines()
.forEach(engine -> node.addEngine(engine));
node.setTreeId(treeId); node.setTreeId(treeId);
switch (entryData.getType()) { switch (entryData.getType()) {
@ -150,16 +156,18 @@ public class DocumentGraphMapper {
private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) { private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector()); return Arrays.stream(atomicTextBlockIds)
.map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
.collect(new TextBlockCollector());
} }
private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) { private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)), return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)), context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
parent, parent,
getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context)); getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
} }
@ -190,8 +198,10 @@ public class DocumentGraphMapper {
this.documentTree = documentTree; this.documentTree = documentTree;
this.pageData = new LinkedList<>(); this.pageData = new LinkedList<>();
this.documentTextData = Arrays.stream(documentData.getDocumentTextData()).toList(); this.documentTextData = Arrays.stream(documentData.getDocumentTextData())
this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData()).toList(); .toList();
this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
.toList();
} }

View File

@ -11,6 +11,7 @@ public abstract class EntityComparators implements Comparator<Entity> {
return new FirstEntity(); return new FirstEntity();
} }
public static class LongestEntity implements Comparator<Entity> { public static class LongestEntity implements Comparator<Entity> {
@Override @Override
@ -27,6 +28,7 @@ public abstract class EntityComparators implements Comparator<Entity> {
return new LongestEntity(); return new LongestEntity();
} }
public static class FirstEntity implements Comparator<Entity> { public static class FirstEntity implements Comparator<Entity> {
@Override @Override

View File

@ -276,7 +276,8 @@ public class EntityCreationService {
"this is some text. a here is more text" and "here is more text". We only want to keep the latter. "this is some text. a here is more text" and "here is more text". We only want to keep the latter.
*/ */
return entityTextRanges.stream() return entityTextRanges.stream()
.filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary))) .filter(boundary -> entityTextRanges.stream()
.noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary)))
.toList(); .toList();
} }
@ -351,10 +352,10 @@ public class EntityCreationService {
return tableNode.streamTableCells() return tableNode.streamTableCells()
.flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()), .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
tableCell, tableCell,
type, type,
entityType, entityType,
tableNode)); tableNode));
} }
@ -362,10 +363,10 @@ public class EntityCreationService {
return tableNode.streamTableCells() return tableNode.streamTableCells()
.flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, tableCell.getTextBlock()), .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, tableCell.getTextBlock()),
tableCell, tableCell,
type, type,
entityType, entityType,
tableNode)); tableNode));
} }
@ -500,7 +501,10 @@ public class EntityCreationService {
public Stream<TextEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) { public Stream<TextEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) {
return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get); return node.streamAllSubNodesOfType(NodeType.PARAGRAPH)
.map(semanticNode -> bySemanticNode(semanticNode, type, entityType))
.filter(Optional::isPresent)
.map(Optional::get);
} }
@ -592,7 +596,11 @@ public class EntityCreationService {
TextRange trimmedTextRange = textRange.trim(node.getTextBlock()); TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node); TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
if (node.getEntities().contains(entity)) { if (node.getEntities().contains(entity)) {
Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny(); Optional<TextEntity> optionalTextEntity = node.getEntities()
.stream()
.filter(e -> e.equals(entity) && e.type().equals(type))
.peek(e -> e.addEngines(engines))
.findAny();
if (optionalTextEntity.isEmpty()) { if (optionalTextEntity.isEmpty()) {
return optionalTextEntity; // Entity has been recategorized and should not be created at all. return optionalTextEntity; // Entity has been recategorized and should not be created at all.
} }
@ -647,17 +655,27 @@ public class EntityCreationService {
return entitiesToMerge.get(0); return entitiesToMerge.get(0);
} }
TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node); TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream()
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); .map(TextEntity::getTextRange)
entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule)); .toList()), type, entityType, node);
mergedEntity.addEngines(entitiesToMerge.stream()
.flatMap(entityNode -> entityNode.getEngines()
.stream())
.collect(Collectors.toSet()));
entitiesToMerge.stream()
.map(TextEntity::getMatchedRuleList)
.flatMap(Collection::stream)
.forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
entitiesToMerge.stream() entitiesToMerge.stream()
.map(TextEntity::getManualOverwrite) .map(TextEntity::getManualOverwrite)
.map(ManualChangeOverwrite::getManualChangeLog) .map(ManualChangeOverwrite::getManualChangeLog)
.flatMap(Collection::stream) .flatMap(Collection::stream)
.forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange)); .forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange));
mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(TextEntity::isDictionaryEntry)); mergedEntity.setDictionaryEntry(entitiesToMerge.stream()
mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(TextEntity::isDossierDictionaryEntry)); .anyMatch(TextEntity::isDictionaryEntry));
mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream()
.anyMatch(TextEntity::isDossierDictionaryEntry));
addEntityToGraph(mergedEntity, node); addEntityToGraph(mergedEntity, node);
insertToKieSession(mergedEntity); insertToKieSession(mergedEntity);
@ -667,7 +685,8 @@ public class EntityCreationService {
public Stream<TextEntity> copyEntities(List<TextEntity> entities, String type, EntityType entityType, SemanticNode node) { public Stream<TextEntity> copyEntities(List<TextEntity> entities, String type, EntityType entityType, SemanticNode node) {
return entities.stream().map(entity -> copyEntity(entity, type, entityType, node)); return entities.stream()
.map(entity -> copyEntity(entity, type, entityType, node));
} }
@ -744,7 +763,8 @@ public class EntityCreationService {
try { try {
if (node.getEntities().contains(entity)) { if (node.getEntities().contains(entity)) {
// If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
node.getEntities().stream()// node.getEntities()
.stream()//
.filter(e -> e.equals(entity))// .filter(e -> e.equals(entity))//
.filter(e -> !e.getTextRange().equals(entity.getTextRange()))// .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
.findAny()// .findAny()//
@ -770,8 +790,10 @@ public class EntityCreationService {
SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes() SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes()
.stream() .stream()
.sorted(Comparator.comparingInt(n -> -n.getTreeId().size())) .sorted(Comparator.comparingInt(n -> -n.getTreeId().size()))
.filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && // .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges()
intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange())) .stream()
.allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
.findFirst() .findFirst()
.orElse(node.getDocumentTree().getRoot().getNode()); .orElse(node.getDocumentTree().getRoot().getNode());
@ -784,7 +806,8 @@ public class EntityCreationService {
return; return;
} }
additionalIntersectingNode.getEntities().add(entityToDuplicate); additionalIntersectingNode.getEntities().add(entityToDuplicate);
additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate)); additionalIntersectingNode.getPages(newTextRange)
.forEach(page -> page.getEntities().add(entityToDuplicate));
entityToDuplicate.addIntersectingNode(additionalIntersectingNode); entityToDuplicate.addIntersectingNode(additionalIntersectingNode);
}); });
} }
@ -806,5 +829,4 @@ public class EntityCreationService {
addEntityToNodeEntitySets(entity); addEntityToNodeEntitySets(entity);
} }
} }

View File

@ -11,7 +11,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
public class EntityCreationUtility { public class EntityCreationUtility {
public static void checkIfBothStartAndEndAreEmpty(String start, String end) { public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
@ -57,7 +56,8 @@ public class EntityCreationUtility {
public static void addEntityToNodeEntitySets(TextEntity entity) { public static void addEntityToNodeEntitySets(TextEntity entity) {
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); entity.getIntersectingNodes()
.forEach(node -> node.getEntities().add(entity));
} }

View File

@ -59,7 +59,9 @@ public class EntityEnrichmentService {
private static List<String> splitToWordsAndRemoveEmptyWords(String textAfter) { private static List<String> splitToWordsAndRemoveEmptyWords(String textAfter) {
return Arrays.stream(textAfter.split(" ")).filter(word -> !Objects.equals("", word)).toList(); return Arrays.stream(textAfter.split(" "))
.filter(word -> !Objects.equals("", word))
.toList();
} }

View File

@ -47,7 +47,9 @@ public class EntityFindingUtility {
} }
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) { public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity,
Map<String, List<TextEntity>> entitiesWithSameValue,
double matchThreshold) {
if (precursorEntity.getValue() == null) { if (precursorEntity.getValue() == null) {
return Optional.empty(); return Optional.empty();
@ -73,11 +75,15 @@ public class EntityFindingUtility {
ClosestEntity closestEntity = optionalClosestEntity.get(); ClosestEntity closestEntity = optionalClosestEntity.get();
if (closestEntity.getDistance() > matchThreshold) { if (closestEntity.getDistance() > matchThreshold) {
log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
precursorEntity.getValue(), precursorEntity.getValue(),
precursorEntity.getEntityPosition().get(0).pageNumber(), precursorEntity.getEntityPosition()
precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), .get(0).pageNumber(),
closestEntity.getDistance(), precursorEntity.getEntityPosition()
matchThreshold); .stream()
.map(RectangleWithPage::rectangle2D)
.toList(),
closestEntity.getDistance(),
matchThreshold);
return Optional.empty(); return Optional.empty();
} }
@ -93,8 +99,14 @@ public class EntityFindingUtility {
private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) { private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet()); Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage()
Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet()); .stream()
.map(PositionOnPage::getPage)
.map(Page::getNumber)
.collect(Collectors.toSet());
Set<Integer> originalPageNumbers = originalPositions.stream()
.map(RectangleWithPage::pageNumber)
.collect(Collectors.toSet());
return entityPageNumbers.containsAll(originalPageNumbers); return entityPageNumbers.containsAll(originalPageNumbers);
} }
@ -105,15 +117,16 @@ public class EntityFindingUtility {
return Double.MAX_VALUE; return Double.MAX_VALUE;
} }
return originalPositions.stream() return originalPositions.stream()
.mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())) .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average()
.average()
.orElse(Double.MAX_VALUE); .orElse(Double.MAX_VALUE);
} }
private static long countRectangles(TextEntity entity) { private static long countRectangles(TextEntity entity) {
return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum(); return entity.getPositionsOnPagePerPage()
.stream()
.mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
} }
@ -144,22 +157,33 @@ public class EntityFindingUtility {
double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY()); double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY());
return Math.abs(minX1 - minX2) // return Math.abs(minX1 - minX2) //
+ Math.abs(minY1 - minY2) // + Math.abs(minY1 - minY2) //
+ Math.abs(maxX1 - maxX2) // + Math.abs(maxX1 - maxX2) //
+ Math.abs(maxY1 - maxY2); + Math.abs(maxY1 - maxY2);
} }
public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) { public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) {
Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet()); Set<Integer> pageNumbers = manualEntities.stream()
Set<String> entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet()); .flatMap(entry -> entry.getEntityPosition()
.stream()
.map(RectangleWithPage::pageNumber))
.collect(Collectors.toSet());
Set<String> entryValues = manualEntities.stream()
.map(PrecursorEntity::getValue)
.filter(Objects::nonNull)
.map(String::toLowerCase)
.collect(Collectors.toSet());
if (!pageNumbers.stream().allMatch(node::onPage)) { if (!pageNumbers.stream()
.allMatch(node::onPage)) {
throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s", throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
node, node,
pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(), pageNumbers.stream()
node.getPages())); .filter(pageNumber -> !node.onPage(pageNumber))
.toList(),
node.getPages()));
} }
SearchImplementation searchImplementation = new SearchImplementation(entryValues, true); SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);

View File

@ -52,9 +52,13 @@ public class ImportedRedactionEntryService {
private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) { private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getForceRedactions(), return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(), manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(), manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(), manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); manualRedactions.getLegalBasisChanges())
.flatMap(Collection::stream)
.map(baseAnnotation -> (BaseAnnotation) baseAnnotation)
.toList();
} }
} }

View File

@ -14,12 +14,14 @@ public class IntersectingNodeVisitor implements NodeVisitor {
private Set<SemanticNode> intersectingNodes; private Set<SemanticNode> intersectingNodes;
private final TextRange textRange; private final TextRange textRange;
public IntersectingNodeVisitor(TextRange textRange) { public IntersectingNodeVisitor(TextRange textRange) {
this.textRange = textRange; this.textRange = textRange;
this.intersectingNodes = new HashSet<>(); this.intersectingNodes = new HashSet<>();
} }
@Override @Override
public void visit(SemanticNode node) { public void visit(SemanticNode node) {

View File

@ -31,7 +31,8 @@ public class ManualRedactionEntryService {
List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList(); List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList();
if (analyzeRequest.getManualRedactions() != null) { if (analyzeRequest.getManualRedactions() != null) {
notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(), notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
document, dossierTemplateId); document,
dossierTemplateId);
log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} }
@ -51,10 +52,13 @@ public class ManualRedactionEntryService {
private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) { private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getForceRedactions(), return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(), manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(), manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(), manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); manualRedactions.getLegalBasisChanges())
.flatMap(Collection::stream)
.map(baseAnnotation -> (BaseAnnotation) baseAnnotation)
.toList();
} }
} }

View File

@ -44,9 +44,11 @@ public class NerEntitiesAdapter {
public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document) { public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document) {
return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(document), return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(document),
nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()), new TextRange(nerEntityModel.getStartOffset(),
nerEntityModel.getType())).toList()); nerEntityModel.getEndOffset()),
nerEntityModel.getType()))
.toList());
} }
@ -83,7 +85,9 @@ public class NerEntitiesAdapter {
List<List<NerEntities.NerEntity>> entityClusters = new LinkedList<>(); List<List<NerEntities.NerEntity>> entityClusters = new LinkedList<>();
List<NerEntities.NerEntity> startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> essentialTypes.contains(e.type())).toList(); List<NerEntities.NerEntity> startEntitiesOfEssentialType = sortedEntities.stream()
.filter(e -> essentialTypes.contains(e.type()))
.toList();
for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) { for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) {
List<NerEntities.NerEntity> currentCluster = new LinkedList<>(); List<NerEntities.NerEntity> currentCluster = new LinkedList<>();
entityClusters.add(currentCluster); entityClusters.add(currentCluster);
@ -105,7 +109,10 @@ public class NerEntitiesAdapter {
} }
} }
return entityClusters.stream().filter(cluster -> cluster.size() >= minPartsToCombine).map(NerEntitiesAdapter::toContainingBoundary).distinct(); return entityClusters.stream()
.filter(cluster -> cluster.size() >= minPartsToCombine)
.map(NerEntitiesAdapter::toContainingBoundary)
.distinct();
} }
@ -124,17 +131,18 @@ public class NerEntitiesAdapter {
public Stream<TextRange> combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) { public Stream<TextRange> combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) {
return combineNerEntities(entityRecognitionEntities, return combineNerEntities(entityRecognitionEntities,
CBI_ADDRESS_ESSENTIAL_TYPES, CBI_ADDRESS_ESSENTIAL_TYPES,
CBI_ADDRESS_TYPES_TO_COMBINE, CBI_ADDRESS_TYPES_TO_COMBINE,
MAX_DISTANCE_BETWEEN_PARTS, MAX_DISTANCE_BETWEEN_PARTS,
MIN_PARTS_TO_COMBINE, MIN_PARTS_TO_COMBINE,
ALLOW_DUPLICATES); ALLOW_DUPLICATES);
} }
private static boolean isDuplicate(List<NerEntities.NerEntity> currentCluster, NerEntities.NerEntity entity, boolean allowDuplicates) { private static boolean isDuplicate(List<NerEntities.NerEntity> currentCluster, NerEntities.NerEntity entity, boolean allowDuplicates) {
return allowDuplicates || currentCluster.stream().anyMatch(e -> e.type().equals(entity.type())); return allowDuplicates || currentCluster.stream()
.anyMatch(e -> e.type().equals(entity.type()));
} }
@ -146,24 +154,34 @@ public class NerEntitiesAdapter {
private static TextRange toContainingBoundary(List<NerEntities.NerEntity> nerEntities) { private static TextRange toContainingBoundary(List<NerEntities.NerEntity> nerEntities) {
return TextRange.merge(nerEntities.stream().map(NerEntities.NerEntity::textRange).toList()); return TextRange.merge(nerEntities.stream()
.map(NerEntities.NerEntity::textRange)
.toList());
} }
private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) { private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
nerEntitiesModel.getData().forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> { nerEntitiesModel.getData()
int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber); .forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> {
entityRecognitionEntity.setStartOffset(newStartOffset); int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber);
entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); entityRecognitionEntity.setStartOffset(newStartOffset);
})); entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
return nerEntitiesModel.getData().values().stream().flatMap(Collection::stream); }));
return nerEntitiesModel.getData().values()
.stream()
.flatMap(Collection::stream);
} }
private static List<Integer> getStringStartOffsetsForMainSections(Document document) { private static List<Integer> getStringStartOffsetsForMainSections(Document document) {
return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getTextRange).map(TextRange::start).toList(); return document.getMainSections()
.stream()
.map(Section::getTextBlock)
.map(TextBlock::getTextRange)
.map(TextRange::start)
.toList();
} }
} }

View File

@ -5,4 +5,5 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
public interface NodeVisitor { public interface NodeVisitor {
void visit(SemanticNode node); void visit(SemanticNode node);
} }

View File

@ -43,7 +43,9 @@ public class PropertiesMapper {
private Rectangle2D parseRectangle2D(String bBox) { private Rectangle2D parseRectangle2D(String bBox) {
List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER)).map(Float::parseFloat).toList(); List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER))
.map(Float::parseFloat)
.toList();
return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
} }

View File

@ -12,6 +12,7 @@ public abstract class SemanticNodeComparators implements Comparator<SemanticNode
return new FirstSemanticNode(); return new FirstSemanticNode();
} }
public static class FirstSemanticNode extends SemanticNodeComparators { public static class FirstSemanticNode extends SemanticNodeComparators {
@Override @Override

View File

@ -50,7 +50,9 @@ public class ComponentDroolsExecutionService {
.filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED)) .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
.map(entry -> Entity.fromEntityLogEntry(entry, document)) .map(entry -> Entity.fromEntityLogEntry(entry, document))
.forEach(kieSession::insert); .forEach(kieSession::insert);
fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); fileAttributes.stream()
.filter(f -> f.getValue() != null)
.forEach(kieSession::insert);
CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> { CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
kieSession.fireAllRules(); kieSession.fireAllRules();
@ -58,7 +60,8 @@ public class ComponentDroolsExecutionService {
}); });
try { try {
completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get(); completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
.get();
} catch (ExecutionException e) { } catch (ExecutionException e) {
kieSession.dispose(); kieSession.dispose();
if (e.getCause() instanceof TimeoutException) { if (e.getCause() instanceof TimeoutException) {
@ -71,7 +74,9 @@ public class ComponentDroolsExecutionService {
} }
List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession); List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
List<Component> components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList(); List<Component> components = getComponents(kieSession).stream()
.sorted(ComponentComparator.first())
.toList();
kieSession.dispose(); kieSession.dispose();
return components; return components;
} }

View File

@ -60,6 +60,7 @@ public class DroolsSyntaxValidationService {
return droolsCompilerSyntaxValidation; return droolsCompilerSyntaxValidation;
} }
private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) {
if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) {
@ -70,13 +71,13 @@ public class DroolsSyntaxValidationService {
String sb = "Following imports are deprecated: \n" + matches.stream() String sb = "Following imports are deprecated: \n" + matches.stream()
.map(m -> imports.substring(m.startIndex(), m.endIndex())) .map(m -> imports.substring(m.startIndex(), m.endIndex()))
.collect(Collectors.joining("\n")); .collect(Collectors.joining("\n"));
return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb) return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build();
.build();
} }
} }
return null; return null;
} }
private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) {
List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>(); List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>();
@ -96,8 +97,7 @@ public class DroolsSyntaxValidationService {
.distinct() .distinct()
.map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm))) .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm)))
.collect(Collectors.joining("\n")); .collect(Collectors.joining("\n"));
warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage) warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage).build());
.build());
} }
} }
} }
@ -160,7 +160,10 @@ public class DroolsSyntaxValidationService {
String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
customSyntaxValidation.getDroolsSyntaxErrorMessages() customSyntaxValidation.getDroolsSyntaxErrorMessages()
.add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup)) .add(DroolsSyntaxErrorMessage.builder()
.line(0)
.column(0)
.message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
.build()); .build());
} }
} }
@ -214,15 +217,13 @@ public class DroolsSyntaxValidationService {
List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream() List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream()
.map(this::buildDroolsSyntaxErrorMessage) .map(this::buildDroolsSyntaxErrorMessage)
.collect(Collectors.toList()); .collect(Collectors.toList());
return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages) return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
.build();
} }
private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) { private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()) return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
.build();
} }
} }

View File

@ -30,8 +30,7 @@ public class KieContainerCreationService {
private final RulesClient rulesClient; private final RulesClient rulesClient;
@Observed(name = "KieContainerCreationService", @Observed(name = "KieContainerCreationService", contextualName = "get-kie-container")
contextualName = "get-kie-container")
public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) { public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) {
try { try {

View File

@ -52,7 +52,12 @@ public class RuleFileParser {
} }
} }
String imports = ruleString.substring(0, packageDescr.getImports().stream().mapToInt(ImportDescr::getEndCharacter).max().orElseThrow() + 1); String imports = ruleString.substring(0,
packageDescr.getImports()
.stream()
.mapToInt(ImportDescr::getEndCharacter)
.max()
.orElseThrow() + 1);
String globals = packageDescr.getGlobals() String globals = packageDescr.getGlobals()
.stream() .stream()
.map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter()))
@ -61,11 +66,20 @@ public class RuleFileParser {
List<RuleClass> ruleClasses = buildRuleClasses(allRules); List<RuleClass> ruleClasses = buildRuleClasses(allRules);
return new RuleFileBluePrint(imports.trim(), return new RuleFileBluePrint(imports.trim(),
packageDescr.getImports().stream().findFirst().map(ImportDescr::getLine).orElse(0), packageDescr.getImports()
globals.trim(), .stream()
packageDescr.getGlobals().stream().findFirst().map(GlobalDescr::getLine).orElse(0), allQueries, .findFirst()
ruleClasses, .map(ImportDescr::getLine)
customDroolsSyntaxValidation); .orElse(0),
globals.trim(),
packageDescr.getGlobals()
.stream()
.findFirst()
.map(GlobalDescr::getLine)
.orElse(0),
allQueries,
ruleClasses,
customDroolsSyntaxValidation);
} }
@ -91,11 +105,12 @@ public class RuleFileParser {
Matcher matcher = ruleIdentifierInCodeFinder.matcher(code); Matcher matcher = ruleIdentifierInCodeFinder.matcher(code);
while (matcher.find()) { while (matcher.find()) {
String identifierInCode = code.substring(matcher.start(1), matcher.end(1)); String identifierInCode = code.substring(matcher.start(1), matcher.end(1));
long line = code.substring(0, matcher.start(1)).lines().count() + lineOffset - 1; long line = code.substring(0, matcher.start(1)).lines()
.count() + lineOffset - 1;
if (!identifier.equals(identifierInCode)) { if (!identifier.equals(identifierInCode)) {
customDroolsSyntaxValidation.addErrorMessage((int) line, customDroolsSyntaxValidation.addErrorMessage((int) line,
0, 0,
String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier)); String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier));
} }
} }
} }
@ -104,23 +119,35 @@ public class RuleFileParser {
private void addDuplicateRuleIdentifierErrorMessage(RuleDescr rule, BasicRule basicRule, DroolsSyntaxValidation customDroolsSyntaxValidation) { private void addDuplicateRuleIdentifierErrorMessage(RuleDescr rule, BasicRule basicRule, DroolsSyntaxValidation customDroolsSyntaxValidation) {
customDroolsSyntaxValidation.addErrorMessage(rule.getLine(), customDroolsSyntaxValidation.addErrorMessage(rule.getLine(),
rule.getColumn(), rule.getColumn(),
String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier())); String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier()));
} }
private List<RuleClass> buildRuleClasses(List<BasicRule> allRules) { private List<RuleClass> buildRuleClasses(List<BasicRule> allRules) {
List<RuleType> ruleTypeOrder = allRules.stream().map(BasicRule::getIdentifier).map(RuleIdentifier::type).distinct().toList(); List<RuleType> ruleTypeOrder = allRules.stream()
Map<RuleType, List<BasicRule>> rulesPerType = allRules.stream().collect(groupingBy(rule -> rule.getIdentifier().type())); .map(BasicRule::getIdentifier)
return ruleTypeOrder.stream().map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))).collect(Collectors.toList()); .map(RuleIdentifier::type)
.distinct()
.toList();
Map<RuleType, List<BasicRule>> rulesPerType = allRules.stream()
.collect(groupingBy(rule -> rule.getIdentifier().type()));
return ruleTypeOrder.stream()
.map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type))))
.collect(Collectors.toList());
} }
private List<RuleUnit> groupingByGroup(List<BasicRule> rules) { private List<RuleUnit> groupingByGroup(List<BasicRule> rules) {
Map<Integer, List<BasicRule>> rulesPerUnit = rules.stream().collect(groupingBy(rule -> rule.getIdentifier().unit())); Map<Integer, List<BasicRule>> rulesPerUnit = rules.stream()
return rulesPerUnit.keySet().stream().sorted().map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))).collect(Collectors.toList()); .collect(groupingBy(rule -> rule.getIdentifier().unit()));
return rulesPerUnit.keySet()
.stream()
.sorted()
.map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit)))
.collect(Collectors.toList());
} }
} }

View File

@ -16,6 +16,7 @@ public class ObservedStorageService {
@Observed(name = "RedactionStorageService", contextualName = "get-document-data") @Observed(name = "RedactionStorageService", contextualName = "get-document-data")
public DocumentData getDocumentData(String dossierId, String fileId) { public DocumentData getDocumentData(String dossierId, String fileId) {
return redactionStorageService.getDocumentData(dossierId, fileId); return redactionStorageService.getDocumentData(dossierId, fileId);
} }

View File

@ -80,9 +80,12 @@ public class RedactionStorageService {
try { try {
ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(), ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactionsPerPage.class); ImportedRedactionsPerPage.class);
return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values().stream().flatMap(List::stream).collect(Collectors.toList())); return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values()
.stream()
.flatMap(List::stream)
.collect(Collectors.toList()));
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available."); log.debug("Imported redactions not available.");
return new ImportedRedactions(); return new ImportedRedactions();
@ -90,14 +93,13 @@ public class RedactionStorageService {
} }
@Timed("redactmanager_getImportedRedactions") @Timed("redactmanager_getImportedRedactions")
public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) { public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) {
try { try {
return storageService.readJSONObject(TenantContext.getTenantId(), return storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactionsPerPage.class); ImportedRedactionsPerPage.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available."); log.debug("Imported redactions not available.");
return null; return null;
@ -111,12 +113,12 @@ public class RedactionStorageService {
try { try {
RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(), RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
RedactionLog.class); RedactionLog.class);
redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry()
.stream() .stream()
.filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
return redactionLog; return redactionLog;
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("RedactionLog not available."); log.debug("RedactionLog not available.");
@ -132,9 +134,9 @@ public class RedactionStorageService {
try { try {
EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
entityLog.setEntityLogEntry(entityLog.getEntityLogEntry() entityLog.setEntityLogEntry(entityLog.getEntityLogEntry()
.stream() .stream()
.filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
return entityLog; return entityLog;
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("EntityLog not available."); log.debug("EntityLog not available.");
@ -156,17 +158,17 @@ public class RedactionStorageService {
try { try {
return DocumentData.builder() return DocumentData.builder()
.documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
DocumentStructure.class)) DocumentStructure.class))
.documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
DocumentTextData[].class)) DocumentTextData[].class))
.documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION),
DocumentPositionData[].class)) DocumentPositionData[].class))
.documentPages(storageService.readJSONObject(TenantContext.getTenantId(), .documentPages(storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
DocumentPage[].class)) DocumentPage[].class))
.build(); .build();
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("DocumentData not available."); log.debug("DocumentData not available.");

View File

@ -11,6 +11,7 @@ public class RuleManagementResources {
private static final String folderPrefix = "drools"; private static final String folderPrefix = "drools";
@SneakyThrows @SneakyThrows
public static InputStream getBaseRuleFileInputStream() { public static InputStream getBaseRuleFileInputStream() {
@ -26,6 +27,7 @@ public class RuleManagementResources {
} }
} }
@SneakyThrows @SneakyThrows
public static InputStream getBaseComponentRuleFileInputStream() { public static InputStream getBaseComponentRuleFileInputStream() {

View File

@ -18,11 +18,11 @@ import lombok.extern.slf4j.Slf4j;
public class DateConverter { public class DateConverter {
static List<SimpleDateFormat> formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH), static List<SimpleDateFormat> formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH),
new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH), new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH),
new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH), new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH),
new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH), new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH),
new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH), new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH),
new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH)); new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
public Optional<Date> parseDate(String dateAsString) { public Optional<Date> parseDate(String dateAsString) {

View File

@ -21,7 +21,9 @@ public final class IdBuilder {
public String buildId(Set<Page> pages, List<Rectangle2D> rectanglesPerLine, String type, String entityType) { public String buildId(Set<Page> pages, List<Rectangle2D> rectanglesPerLine, String type, String entityType) {
return buildId(pages.stream().map(Page::getNumber).collect(Collectors.toList()), rectanglesPerLine, type, entityType); return buildId(pages.stream()
.map(Page::getNumber)
.collect(Collectors.toList()), rectanglesPerLine, type, entityType);
} }
@ -29,7 +31,9 @@ public final class IdBuilder {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append(type).append(entityType); sb.append(type).append(entityType);
List<Integer> sortedPageNumbers = pageNumbers.stream().sorted(Comparator.comparingInt(Integer::intValue)).toList(); List<Integer> sortedPageNumbers = pageNumbers.stream()
.sorted(Comparator.comparingInt(Integer::intValue))
.toList();
sortedPageNumbers.forEach(sb::append); sortedPageNumbers.forEach(sb::append);
rectanglesPerLine.forEach(rectangle2D -> sb.append(Math.round(rectangle2D.getX())) rectanglesPerLine.forEach(rectangle2D -> sb.append(Math.round(rectangle2D.getX()))
.append(Math.round(rectangle2D.getY())) .append(Math.round(rectangle2D.getY()))

View File

@ -22,19 +22,25 @@ public class RectangleTransformations {
public static Rectangle2D atomicTextBlockBBox(List<AtomicTextBlock> atomicTextBlocks) { public static Rectangle2D atomicTextBlockBBox(List<AtomicTextBlock> atomicTextBlocks) {
return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector()); return atomicTextBlocks.stream()
.flatMap(atomicTextBlock -> atomicTextBlock.getPositions()
.stream())
.collect(new Rectangle2DBBoxCollector());
} }
public static Rectangle2D rectangleBBox(List<Position> positions) { public static Rectangle2D rectangleBBox(List<Position> positions) {
return positions.stream().map(Position::toRectangle2D).collect(new Rectangle2DBBoxCollector()); return positions.stream()
.map(Position::toRectangle2D)
.collect(new Rectangle2DBBoxCollector());
} }
public static Rectangle2D rectangle2DBBox(List<Rectangle2D> rectangle2DList) { public static Rectangle2D rectangle2DBBox(List<Rectangle2D> rectangle2DList) {
return rectangle2DList.stream().collect(new Rectangle2DBBoxCollector()); return rectangle2DList.stream()
.collect(new Rectangle2DBBoxCollector());
} }
@ -49,7 +55,9 @@ public class RectangleTransformations {
if (rectangle2DList.isEmpty()) { if (rectangle2DList.isEmpty()) {
return Collections.emptyList(); return Collections.emptyList();
} }
double splitThreshold = rectangle2DList.stream().mapToDouble(RectangularShape::getWidth).average().orElse(5) * 5.0; double splitThreshold = rectangle2DList.stream()
.mapToDouble(RectangularShape::getWidth).average()
.orElse(5) * 5.0;
List<List<Rectangle2D>> rectangleListsWithGaps = new LinkedList<>(); List<List<Rectangle2D>> rectangleListsWithGaps = new LinkedList<>();
List<Rectangle2D> rectangleListWithoutGaps = new LinkedList<>(); List<Rectangle2D> rectangleListWithoutGaps = new LinkedList<>();
@ -66,7 +74,9 @@ public class RectangleTransformations {
previousRectangle = currentRectangle; previousRectangle = currentRectangle;
} }
} }
return rectangleListsWithGaps.stream().map(RectangleTransformations::rectangle2DBBox).toList(); return rectangleListsWithGaps.stream()
.map(RectangleTransformations::rectangle2DBBox)
.toList();
} }
@ -96,9 +106,9 @@ public class RectangleTransformations {
public BinaryOperator<BBox> combiner() { public BinaryOperator<BBox> combiner() {
return (b1, b2) -> new BBox(Math.min(b1.lowerLeftX, b2.lowerLeftX), return (b1, b2) -> new BBox(Math.min(b1.lowerLeftX, b2.lowerLeftX),
Math.min(b1.lowerLeftY, b2.lowerLeftY), Math.min(b1.lowerLeftY, b2.lowerLeftY),
Math.max(b1.upperRightX, b2.upperRightX), Math.max(b1.upperRightX, b2.upperRightX),
Math.max(b1.upperRightY, b2.upperRightY)); Math.max(b1.upperRightY, b2.upperRightY));
} }

View File

@ -89,7 +89,10 @@ public class RedactionSearchUtility {
public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) { public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) {
List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed().map(textBlock::getLineTextRange).filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary)).toList(); List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
.map(textBlock::getLineTextRange)
.filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary))
.toList();
if (lineBoundaries.isEmpty()) { if (lineBoundaries.isEmpty()) {
return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start()); return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
} }
@ -132,6 +135,7 @@ public class RedactionSearchUtility {
return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern); return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern);
} }
public static List<TextRange> findTextRangesByRegexIgnoreCase(String regexPattern, TextBlock textBlock) { public static List<TextRange> findTextRangesByRegexIgnoreCase(String regexPattern, TextBlock textBlock) {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, true); Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
@ -185,19 +189,21 @@ public class RedactionSearchUtility {
return getTextRangesByPattern(textBlock, 0, pattern); return getTextRangesByPattern(textBlock, 0, pattern);
} }
public static List<TextRange> findTextRangesByList(List<String> searchList, TextBlock textBlock) { public static List<TextRange> findTextRangesByList(List<String> searchList, TextBlock textBlock) {
List<TextRange> boundaries = new LinkedList<>(); List<TextRange> boundaries = new LinkedList<>();
for (var searchString: searchList) { for (var searchString : searchList) {
boundaries.addAll(findTextRangesByString(searchString, textBlock)); boundaries.addAll(findTextRangesByString(searchString, textBlock));
} }
return boundaries; return boundaries;
} }
public static List<TextRange> findTextRangesByListIgnoreCase(List<String> searchList, TextBlock textBlock) { public static List<TextRange> findTextRangesByListIgnoreCase(List<String> searchList, TextBlock textBlock) {
List<TextRange> boundaries = new LinkedList<>(); List<TextRange> boundaries = new LinkedList<>();
for (var searchString: searchList) { for (var searchString : searchList) {
boundaries.addAll(findTextRangesByStringIgnoreCase(searchString, textBlock)); boundaries.addAll(findTextRangesByStringIgnoreCase(searchString, textBlock));
} }
return boundaries; return boundaries;

View File

@ -20,7 +20,8 @@ public final class ResourceLoader {
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
} }
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
return br.lines().collect(Collectors.toSet()); return br.lines()
.collect(Collectors.toSet());
} catch (IOException e) { } catch (IOException e) {
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
} }

View File

@ -54,16 +54,16 @@ public final class SeparatorUtils {
private static boolean validateEnd(TextBlock textBlock, TextRange textRange) { private static boolean validateEnd(TextBlock textBlock, TextRange textRange) {
return textRange.end() == textBlock.getTextRange().end() ||// return textRange.end() == textBlock.getTextRange().end() ||//
SeparatorUtils.isSeparator(textBlock.charAt(textRange.end())) ||// SeparatorUtils.isSeparator(textBlock.charAt(textRange.end())) ||//
SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.end() - 1)); SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.end() - 1));
} }
private static boolean validateStart(TextBlock textBlock, TextRange textRange) { private static boolean validateStart(TextBlock textBlock, TextRange textRange) {
return textRange.start() == textBlock.getTextRange().start() ||// return textRange.start() == textBlock.getTextRange().start() ||//
SeparatorUtils.isSeparator(textBlock.charAt(textRange.start() - 1)) ||// SeparatorUtils.isSeparator(textBlock.charAt(textRange.start() - 1)) ||//
SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.start())); SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.start()));
} }
} }

View File

@ -172,7 +172,7 @@ public abstract class AbstractRedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR,
false)); false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_PII, false)); when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_PII, false));
when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY_INDICATOR, false));
@ -183,9 +183,9 @@ public abstract class AbstractRedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR, when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,
true)); true));
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR, when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
true)); true));
} }
@ -193,45 +193,102 @@ public abstract class AbstractRedactionIntegrationTest {
protected void loadDictionaryForTest() { protected void loadDictionaryForTest() {
dictionary.computeIfAbsent(DICTIONARY_AUTHOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_AUTHOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(DICTIONARY_SPONSOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_SPONSOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(VERTEBRATE_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(VERTEBRATE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(DICTIONARY_ADDRESS, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_ADDRESS, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(HINT_ONLY_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(HINT_ONLY_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(MUST_REDACT_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(MUST_REDACT_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PUBLISHED_INFORMATION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(PUBLISHED_INFORMATION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/published_information.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(TEST_METHOD_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(TEST_METHOD_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/test_method.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) dictionary.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/PII.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PURITY_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(PURITY_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/purity.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(IMAGE_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(IMAGE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(OCR_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(OCR_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(LOGO_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(LOGO_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(SIGNATURE_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(SIGNATURE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(FORMULA_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(FORMULA_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS_INDICATOR, v -> new ArrayList<>()) dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>()); dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
} }
@ -240,7 +297,10 @@ public abstract class AbstractRedactionIntegrationTest {
dictionary.clear(); dictionary.clear();
dictionary.computeIfAbsent(ROTATE_SIMPLE_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(ROTATE_SIMPLE_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
} }
@ -373,8 +433,8 @@ public abstract class AbstractRedactionIntegrationTest {
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream()); responseJson.getInputStream());
} }
@ -424,7 +484,9 @@ public abstract class AbstractRedactionIntegrationTest {
if (entries == null) { if (entries == null) {
return Collections.emptyList(); return Collections.emptyList();
} }
return entries.stream().map(this::toDictionaryEntry).collect(Collectors.toList()); return entries.stream()
.map(this::toDictionaryEntry)
.collect(Collectors.toList());
} }
@ -480,11 +542,11 @@ public abstract class AbstractRedactionIntegrationTest {
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
cvServiceResponseFileStream); cvServiceResponseFileStream);
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseStream); imageServiceResponseStream);
return request; return request;

View File

@ -104,15 +104,15 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);

View File

@ -82,18 +82,18 @@ public class DictionaryServiceTest {
when(dictionaryClient.getVersion(anyString())).thenReturn(0L); when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc")); "#cccccc"));
var type1 = new Type(); var type1 = new Type();
type1.setType("type1"); type1.setType("type1");
@ -101,7 +101,9 @@ public class DictionaryServiceTest {
type1.setVersion(1L); type1.setVersion(1L);
type1.setHexColor("#cccccc"); type1.setHexColor("#cccccc");
type1.setHasDictionary(true); type1.setHasDictionary(true);
type1.setEntries(Stream.of("a", "b", "c").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList())); type1.setEntries(Stream.of("a", "b", "c")
.map(t -> new DictionaryEntry(1, t, 1L, false, "type1"))
.collect(Collectors.toList()));
var type2 = new Type(); var type2 = new Type();
type2.setType("type2"); type2.setType("type2");
@ -109,7 +111,9 @@ public class DictionaryServiceTest {
type2.setVersion(1L); type2.setVersion(1L);
type2.setHexColor("#cccccc"); type2.setHexColor("#cccccc");
type2.setHasDictionary(true); type2.setHasDictionary(true);
type2.setEntries(Stream.of("d", "e", "f").map(t -> new DictionaryEntry(1, t, 1L, false, "type2")).collect(Collectors.toList())); type2.setEntries(Stream.of("d", "e", "f")
.map(t -> new DictionaryEntry(1, t, 1L, false, "type2"))
.collect(Collectors.toList()));
var type1Updated = new Type(); var type1Updated = new Type();
type1Updated.setType("type1"); type1Updated.setType("type1");
@ -117,7 +121,9 @@ public class DictionaryServiceTest {
type1Updated.setVersion(2L); type1Updated.setVersion(2L);
type1Updated.setHexColor("#cccccc"); type1Updated.setHexColor("#cccccc");
type1Updated.setHasDictionary(true); type1Updated.setHasDictionary(true);
type1Updated.setEntries(Stream.of("z", "q", "x").map(t -> new DictionaryEntry(1, t, 2L, false, "type1")).collect(Collectors.toList())); type1Updated.setEntries(Stream.of("z", "q", "x")
.map(t -> new DictionaryEntry(1, t, 2L, false, "type1"))
.collect(Collectors.toList()));
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1, type2)); when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1, type2));
when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(type1); when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(type1);
@ -174,18 +180,18 @@ public class DictionaryServiceTest {
when(dictionaryClient.getVersion(anyString())).thenReturn(0L); when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dtId", when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dtId",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc", "#cccccc",
"#cccccc")); "#cccccc"));
var type = "type"; var type = "type";
var dtType = new Type(); var dtType = new Type();
@ -195,7 +201,9 @@ public class DictionaryServiceTest {
dtType.setHexColor("#cccccc"); dtType.setHexColor("#cccccc");
dtType.setHasDictionary(true); dtType.setHasDictionary(true);
dtType.setDossierTemplateId("dtId"); dtType.setDossierTemplateId("dtId");
List<DictionaryEntry> dossierTemplateEntries = Stream.of("aa", "bb").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList()); List<DictionaryEntry> dossierTemplateEntries = Stream.of("aa", "bb")
.map(t -> new DictionaryEntry(1, t, 1L, false, "type1"))
.collect(Collectors.toList());
dossierTemplateEntries.add(new DictionaryEntry(1, "cc", 2L, false, "type1")); dossierTemplateEntries.add(new DictionaryEntry(1, "cc", 2L, false, "type1"));
assertThat(dossierTemplateEntries.size()).isEqualTo(3); assertThat(dossierTemplateEntries.size()).isEqualTo(3);
dtType.setEntries(dossierTemplateEntries); dtType.setEntries(dossierTemplateEntries);
@ -208,7 +216,9 @@ public class DictionaryServiceTest {
dossierType.setHasDictionary(true); dossierType.setHasDictionary(true);
dossierType.setDossierTemplateId("dtId"); dossierType.setDossierTemplateId("dtId");
dossierType.setDossierId("dossierId"); dossierType.setDossierId("dossierId");
List<DictionaryEntry> dossierEntries = Stream.of("aa", "bb").map(t -> new DictionaryEntry(1, t, 2L, true, "dossierType")).collect(Collectors.toList()); List<DictionaryEntry> dossierEntries = Stream.of("aa", "bb")
.map(t -> new DictionaryEntry(1, t, 2L, true, "dossierType"))
.collect(Collectors.toList());
dossierEntries.add(new DictionaryEntry(1, "dd", 1L, false, "dossierType")); dossierEntries.add(new DictionaryEntry(1, "dd", 1L, false, "dossierType"));
assertThat(dossierEntries.size()).isEqualTo(3); assertThat(dossierEntries.size()).isEqualTo(3);
dossierType.setEntries(dossierEntries); dossierType.setEntries(dossierEntries);
@ -224,15 +234,17 @@ public class DictionaryServiceTest {
dictionaryService.updateDictionary("dtId", "dossierId"); dictionaryService.updateDictionary("dtId", "dossierId");
var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId"); var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId");
assertThat(dict.getDictionaryModels().size()).isEqualTo(1); assertThat(dict.getDictionaryModels().size()).isEqualTo(1);
var dictModel = dict.getDictionaryModels().get(0); var dictModel = dict.getDictionaryModels()
.get(0);
assertThat(dictModel.getType()).isEqualTo(type); assertThat(dictModel.getType()).isEqualTo(type);
assertThat(dictModel.getEntries().size()).isEqualTo(4); assertThat(dictModel.getEntries().size()).isEqualTo(4);
dictModel.getEntries().forEach(entry -> { dictModel.getEntries()
switch (entry.getValue()) { .forEach(entry -> {
case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId()); switch (entry.getValue()) {
case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()); case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId());
} case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId());
}); }
});
} }
} }

View File

@ -8,6 +8,7 @@ import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -55,9 +56,11 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf"); AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
ClassPathResource importedRedactionClasspathResource = new ClassPathResource("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json"); ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactionClasspathResource.getInputStream()); "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactionClasspathResource.getInputStream());
// AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf", // AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf",
// "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json"); // "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json");
@ -86,7 +89,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
// FIXME TableNodeFactory: 36, why has table no rows/cols here. // FIXME TableNodeFactory: 36, why has table no rows/cols here.
AnalyzeRequest request = prepareStorage("files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.pdf", AnalyzeRequest request = prepareStorage("files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.pdf",
"files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.TABLES.json"); "files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.TABLES.json");
System.out.println("Start Full integration test"); System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request); analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
@ -166,15 +169,15 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);

View File

@ -87,15 +87,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
@ -122,6 +122,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A."); assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A.");
} }
@Test @Test
public void acceptanceTests() throws IOException { public void acceptanceTests() throws IOException {
@ -133,8 +134,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis"); System.out.println("Finished analysis");
EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow(); .orElseThrow();
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
@ -146,8 +149,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow(); .orElseThrow();
var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst()
.orElseThrow();
assertEquals(EntryState.APPLIED, asyaLyon2.getState()); assertEquals(EntryState.APPLIED, asyaLyon2.getState());
@ -168,13 +173,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
.stream() .stream()
.filter(entry -> entry.getType().equals(type)) .filter(entry -> entry.getType().equals(type))
.filter(entry -> entry.getValue().equals(value)) .filter(entry -> entry.getValue().equals(value))
.filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0))); .filter(entry -> entry.getContainingNodeId()
.get(0).equals(sectionNumber.get(0)));
} }
private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) { private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) {
return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value)); return redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getType().equals(type))
.filter(entry -> entry.getValue().equals(value));
} }
@ -201,7 +210,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(EntryState.IGNORED, assertEquals(EntryState.IGNORED,
findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState()); findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY))
.findFirst()
.get().getState());
} }

View File

@ -78,15 +78,15 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
@ -128,7 +128,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(1); assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(1);
EntityLogEntry redactionLogEntry = entityLog.getEntityLogEntry().get(0); EntityLogEntry redactionLogEntry = entityLog.getEntityLogEntry()
.get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorAndPIIDictionary); assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorAndPIIDictionary);
@ -164,7 +165,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
assertThat(redactionLog.getEntityLogEntry().size()).isEqualTo(1); assertThat(redactionLog.getEntityLogEntry().size()).isEqualTo(1);
EntityLogEntry redactionLogEntry = redactionLog.getEntityLogEntry().get(0); EntityLogEntry redactionLogEntry = redactionLog.getEntityLogEntry()
.get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorDictionary); assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorDictionary);

View File

@ -136,99 +136,99 @@ public class RulesTest {
private final Map<String, List<String>> dossierDictionary = new HashMap<>(); private final Map<String, List<String>> dossierDictionary = new HashMap<>();
private final Map<String, List<String>> falsePositive = new HashMap<>(); private final Map<String, List<String>> falsePositive = new HashMap<>();
private static final Map<String, String> typeColorMap = Map.ofEntries(entry(VERTEBRATE, "#ff85f7"), private static final Map<String, String> typeColorMap = Map.ofEntries(entry(VERTEBRATE, "#ff85f7"),
entry(ADDRESS, "#ffe187"), entry(ADDRESS, "#ffe187"),
entry(AUTHOR, "#ffe187"), entry(AUTHOR, "#ffe187"),
entry(SPONSOR, "#85ebff"), entry(SPONSOR, "#85ebff"),
entry(NO_REDACTION_INDICATOR, "#be85ff"), entry(NO_REDACTION_INDICATOR, "#be85ff"),
entry(REDACTION_INDICATOR, "#caff85"), entry(REDACTION_INDICATOR, "#caff85"),
entry(HINT_ONLY, "#abc0c4"), entry(HINT_ONLY, "#abc0c4"),
entry(MUST_REDACT, "#fab4c0"), entry(MUST_REDACT, "#fab4c0"),
entry(PUBLISHED_INFORMATION, "#85ebff"), entry(PUBLISHED_INFORMATION, "#85ebff"),
entry(TEST_METHOD, "#91fae8"), entry(TEST_METHOD, "#91fae8"),
entry(PII, "#66ccff"), entry(PII, "#66ccff"),
entry(PURITY, "#ffe187"), entry(PURITY, "#ffe187"),
entry(IMAGE, "#fcc5fb"), entry(IMAGE, "#fcc5fb"),
entry(OCR, "#fcc5fb"), entry(OCR, "#fcc5fb"),
entry(LOGO, "#ffe187"), entry(LOGO, "#ffe187"),
entry(FORMULA, "#ffe187"), entry(FORMULA, "#ffe187"),
entry(SIGNATURE, "#ffe187"), entry(SIGNATURE, "#ffe187"),
entry(IMPORTED_REDACTION, "#fcfbe6")); entry(IMPORTED_REDACTION, "#fcfbe6"));
private static final Map<String, Boolean> hintTypeMap = Map.ofEntries(entry(VERTEBRATE, true), private static final Map<String, Boolean> hintTypeMap = Map.ofEntries(entry(VERTEBRATE, true),
entry(ADDRESS, false), entry(ADDRESS, false),
entry(AUTHOR, false), entry(AUTHOR, false),
entry(SPONSOR, false), entry(SPONSOR, false),
entry(NO_REDACTION_INDICATOR, true), entry(NO_REDACTION_INDICATOR, true),
entry(REDACTION_INDICATOR, true), entry(REDACTION_INDICATOR, true),
entry(HINT_ONLY, true), entry(HINT_ONLY, true),
entry(MUST_REDACT, true), entry(MUST_REDACT, true),
entry(PUBLISHED_INFORMATION, true), entry(PUBLISHED_INFORMATION, true),
entry(TEST_METHOD, true), entry(TEST_METHOD, true),
entry(PII, false), entry(PII, false),
entry(PURITY, false), entry(PURITY, false),
entry(IMAGE, true), entry(IMAGE, true),
entry(OCR, true), entry(OCR, true),
entry(FORMULA, false), entry(FORMULA, false),
entry(LOGO, false), entry(LOGO, false),
entry(SIGNATURE, false), entry(SIGNATURE, false),
entry(DOSSIER_REDACTIONS, false), entry(DOSSIER_REDACTIONS, false),
entry(IMPORTED_REDACTION, false)); entry(IMPORTED_REDACTION, false));
private static final Map<String, Boolean> caseInSensitiveMap = Map.ofEntries(entry(VERTEBRATE, true), private static final Map<String, Boolean> caseInSensitiveMap = Map.ofEntries(entry(VERTEBRATE, true),
entry(ADDRESS, false), entry(ADDRESS, false),
entry(AUTHOR, false), entry(AUTHOR, false),
entry(SPONSOR, false), entry(SPONSOR, false),
entry(NO_REDACTION_INDICATOR, true), entry(NO_REDACTION_INDICATOR, true),
entry(REDACTION_INDICATOR, true), entry(REDACTION_INDICATOR, true),
entry(HINT_ONLY, true), entry(HINT_ONLY, true),
entry(MUST_REDACT, true), entry(MUST_REDACT, true),
entry(PUBLISHED_INFORMATION, true), entry(PUBLISHED_INFORMATION, true),
entry(TEST_METHOD, false), entry(TEST_METHOD, false),
entry(PII, false), entry(PII, false),
entry(PURITY, false), entry(PURITY, false),
entry(IMAGE, true), entry(IMAGE, true),
entry(OCR, true), entry(OCR, true),
entry(SIGNATURE, true), entry(SIGNATURE, true),
entry(LOGO, true), entry(LOGO, true),
entry(FORMULA, true), entry(FORMULA, true),
entry(DOSSIER_REDACTIONS, false), entry(DOSSIER_REDACTIONS, false),
entry(IMPORTED_REDACTION, false)); entry(IMPORTED_REDACTION, false));
private static final Map<String, Boolean> recommendationTypeMap = Map.ofEntries(entry(VERTEBRATE, false), private static final Map<String, Boolean> recommendationTypeMap = Map.ofEntries(entry(VERTEBRATE, false),
entry(ADDRESS, false), entry(ADDRESS, false),
entry(AUTHOR, false), entry(AUTHOR, false),
entry(SPONSOR, false), entry(SPONSOR, false),
entry(NO_REDACTION_INDICATOR, false), entry(NO_REDACTION_INDICATOR, false),
entry(REDACTION_INDICATOR, false), entry(REDACTION_INDICATOR, false),
entry(HINT_ONLY, false), entry(HINT_ONLY, false),
entry(MUST_REDACT, false), entry(MUST_REDACT, false),
entry(PUBLISHED_INFORMATION, false), entry(PUBLISHED_INFORMATION, false),
entry(TEST_METHOD, false), entry(TEST_METHOD, false),
entry(PII, false), entry(PII, false),
entry(PURITY, false), entry(PURITY, false),
entry(IMAGE, false), entry(IMAGE, false),
entry(OCR, false), entry(OCR, false),
entry(FORMULA, false), entry(FORMULA, false),
entry(SIGNATURE, false), entry(SIGNATURE, false),
entry(LOGO, false), entry(LOGO, false),
entry(DOSSIER_REDACTIONS, false), entry(DOSSIER_REDACTIONS, false),
entry(IMPORTED_REDACTION, false)); entry(IMPORTED_REDACTION, false));
private static final Map<String, Integer> rankTypeMap = Map.ofEntries(entry(PURITY, 155), private static final Map<String, Integer> rankTypeMap = Map.ofEntries(entry(PURITY, 155),
entry(PII, 150), entry(PII, 150),
entry(ADDRESS, 140), entry(ADDRESS, 140),
entry(AUTHOR, 130), entry(AUTHOR, 130),
entry(SPONSOR, 120), entry(SPONSOR, 120),
entry(VERTEBRATE, 110), entry(VERTEBRATE, 110),
entry(MUST_REDACT, 100), entry(MUST_REDACT, 100),
entry(REDACTION_INDICATOR, 90), entry(REDACTION_INDICATOR, 90),
entry(NO_REDACTION_INDICATOR, 80), entry(NO_REDACTION_INDICATOR, 80),
entry(PUBLISHED_INFORMATION, 70), entry(PUBLISHED_INFORMATION, 70),
entry(TEST_METHOD, 60), entry(TEST_METHOD, 60),
entry(HINT_ONLY, 50), entry(HINT_ONLY, 50),
entry(IMAGE, 30), entry(IMAGE, 30),
entry(OCR, 29), entry(OCR, 29),
entry(LOGO, 28), entry(LOGO, 28),
entry(SIGNATURE, 27), entry(SIGNATURE, 27),
entry(FORMULA, 26), entry(FORMULA, 26),
entry(DOSSIER_REDACTIONS, 200), entry(DOSSIER_REDACTIONS, 200),
entry(IMPORTED_REDACTION, 200)); entry(IMPORTED_REDACTION, 200));
private final Colors colors = new Colors(); private final Colors colors = new Colors();
@Autowired @Autowired
@ -273,15 +273,15 @@ public class RulesTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS) .type(DOSSIER_REDACTIONS)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
mockDictionaryCalls(0L); mockDictionaryCalls(0L);
@ -372,10 +372,10 @@ public class RulesTest {
log.warn("WARNING: {} files from {} failed", failedFiles.size(), fileSize); log.warn("WARNING: {} files from {} failed", failedFiles.size(), fileSize);
for (String fileName : failedFiles.keySet()) { for (String fileName : failedFiles.keySet()) {
log.warn(" - '{}' failed with Error: {} See line {} in {}", log.warn(" - '{}' failed with Error: {} See line {} in {}",
fileName, fileName,
failedFiles.get(fileName), failedFiles.get(fileName),
failedFiles.get(fileName).getStackTrace()[0].getLineNumber(), failedFiles.get(fileName).getStackTrace()[0].getLineNumber(),
failedFiles.get(fileName).getStackTrace()[0].getClassName()); failedFiles.get(fileName).getStackTrace()[0].getClassName());
} }
} }
@ -442,9 +442,13 @@ public class RulesTest {
assertThat(entityLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion()); assertThat(entityLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
assertThat(entityLog.getEntityLogEntry() assertThat(entityLog.getEntityLogEntry()
.stream().filter(r -> !r.getEntryType().equals(EntryType.FALSE_POSITIVE)).filter(r -> !r.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) .stream()
.collect(Collectors.toSet()) .filter(r -> !r.getEntryType().equals(EntryType.FALSE_POSITIVE))
.size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size()); .filter(r -> !r.getEntryType().equals(EntryType.FALSE_RECOMMENDATION))
.collect(Collectors.toSet()).size()).isEqualTo(savedRedactionLog.getRedactionLogEntry()
.stream()
.filter(r -> !r.isFalsePositive())
.collect(Collectors.toSet()).size());
assertThat(entityLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size()); assertThat(entityLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
for (EntityLogLegalBasis redactionLegalBasis : entityLog.getLegalBasis()) { for (EntityLogLegalBasis redactionLegalBasis : entityLog.getLegalBasis()) {
@ -458,7 +462,10 @@ public class RulesTest {
} }
for (EntityLogEntry redactionLogEntry : entityLog.getEntityLogEntry()) { for (EntityLogEntry redactionLogEntry : entityLog.getEntityLogEntry()) {
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst(); var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry()
.stream()
.filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId()))
.findFirst();
assertThat(savedRedactionLogEntry).isPresent(); assertThat(savedRedactionLogEntry).isPresent();
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId()); assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType()); assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
@ -473,7 +480,8 @@ public class RulesTest {
assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)); assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE));
assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection()); assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor()); assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId().get(0)); assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId()
.get(0));
assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore()); assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter()); assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset()); assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
@ -485,8 +493,7 @@ public class RulesTest {
assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded()); assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
for (Position position : redactionLogEntry.getPositions()) { for (Position position : redactionLogEntry.getPositions()) {
var savedRectangle = savedRedactionLogEntry.get() var savedRectangle = savedRedactionLogEntry.get().getPositions()
.getPositions()
.stream() .stream()
.filter(r -> r.getPage() == position.getPageNumber()) .filter(r -> r.getPage() == position.getPageNumber())
.filter(r -> r.getTopLeft().getX() == position.x()) .filter(r -> r.getTopLeft().getX() == position.x())
@ -498,17 +505,16 @@ public class RulesTest {
} }
for (Change change : redactionLogEntry.getChanges()) { for (Change change : redactionLogEntry.getChanges()) {
var savedChange = savedRedactionLogEntry.get() var savedChange = savedRedactionLogEntry.get().getChanges()
.getChanges()
.stream() .stream()
.filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber()).filter(c -> c.getType().name().equals(change.getType().name())) .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
.filter(c -> c.getType().name().equals(change.getType().name()))
.findFirst(); .findFirst();
assertThat(savedChange).isPresent(); assertThat(savedChange).isPresent();
} }
for (ManualChange manualChange : redactionLogEntry.getManualChanges()) { for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
var savedManualChange = savedRedactionLogEntry.get() var savedManualChange = savedRedactionLogEntry.get().getManualChanges()
.getManualChanges()
.stream() .stream()
.filter(m -> m.getManualRedactionType().name().equals(manualChange.getManualRedactionType().name())) .filter(m -> m.getManualRedactionType().name().equals(manualChange.getManualRedactionType().name()))
.filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId())) .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
@ -517,7 +523,8 @@ public class RulesTest {
assertThat(savedManualChange).isPresent(); assertThat(savedManualChange).isPresent();
} }
assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(Engine[]::new)); assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines()
.toArray(Engine[]::new));
assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference()); assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections()); assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
@ -622,8 +629,8 @@ public class RulesTest {
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream()); responseJson.getInputStream());
} }
@ -647,8 +654,8 @@ public class RulesTest {
.build(); .build();
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
new ClassPathResource("files/cv_service_empty_response.json").getInputStream()); new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream); storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
return request; return request;
@ -659,45 +666,102 @@ public class RulesTest {
private void loadDictionaryForTest() { private void loadDictionaryForTest() {
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/published_information.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/test_method.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/PII.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/purity.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/empty.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
} }
@ -736,12 +800,12 @@ public class RulesTest {
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false)); when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false)); when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR,
false)); false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION,
false)); false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false)); when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false)); when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
@ -809,7 +873,10 @@ public class RulesTest {
protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType) { protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType) {
return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType,
AnalyzeRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build())); AnalyzeRequest.builder()
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build()));
} }

View File

@ -1,6 +1,5 @@
package com.iqser.red.service.redaction.v1.server.annotate; package com.iqser.red.service.redaction.v1.server.annotate;
import java.util.List; import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;

View File

@ -79,8 +79,7 @@ public class AnnotationService {
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
pdDocument.save(byteArrayOutputStream); pdDocument.save(byteArrayOutputStream);
return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()) return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
.build();
} }
} }
} }

View File

@ -13,10 +13,7 @@ public class TextEntityTest {
@Test @Test
public void testMatchedRule() { public void testMatchedRule() {
PrecursorEntity entity = PrecursorEntity.builder() PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build();
.type("PII")
.entityType(EntityType.ENTITY)
.build();
entity.skip("CBI.1.0", ""); entity.skip("CBI.1.0", "");
entity.skip("CBI.2.0", ""); entity.skip("CBI.2.0", "");
entity.skip("CBI.3.0", ""); entity.skip("CBI.3.0", "");
@ -27,14 +24,10 @@ public class TextEntityTest {
} }
@Test @Test
public void testMatchedRuleWithNonsense() { public void testMatchedRuleWithNonsense() {
PrecursorEntity entity = PrecursorEntity.builder() PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build();
.type("PII")
.entityType(EntityType.ENTITY)
.build();
assertThrows(IllegalArgumentException.class, () -> { assertThrows(IllegalArgumentException.class, () -> {
entity.skip("", ""); entity.skip("", "");
}); });

View File

@ -49,19 +49,26 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
} }
@Test @Test
public void assertEntitiesAreDuplicatedWithTheirTableCell() { public void assertEntitiesAreDuplicatedWithTheirTableCell() {
Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf"); Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf");
List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document).toList(); List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document)
.toList();
assertEquals(3, entities.size()); assertEquals(3, entities.size());
assertEquals(1, entities.stream().distinct().count()); assertEquals(1,
entities.stream()
.distinct()
.count());
assertEquals(2, entities.get(0).getDuplicateTextRanges().size()); assertEquals(2, entities.get(0).getDuplicateTextRanges().size());
var node = entities.get(0).getDeepestFullyContainingNode(); var node = entities.get(0).getDeepestFullyContainingNode();
assertTrue(node.getTextRange().contains(entities.get(0).getTextRange())); assertTrue(node.getTextRange().contains(entities.get(0).getTextRange()));
assertTrue(entities.get(0).getDuplicateTextRanges().stream().allMatch(tr -> node.getTextRange().contains(tr))); assertTrue(entities.get(0).getDuplicateTextRanges()
.stream()
.allMatch(tr -> node.getTextRange().contains(tr)));
} }
@ -84,8 +91,10 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
Document document = buildGraph("files/new/crafted document.pdf"); Document document = buildGraph("files/new/crafted document.pdf");
String type = "CBI_author"; String type = "CBI_author";
assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document)
assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); .isPresent());
assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document)
.isPresent());
assertEquals(1, document.getEntities().size()); assertEquals(1, document.getEntities().size());
verify(kieSession, times(1)).insert(any(TextEntity.class)); verify(kieSession, times(1)).insert(any(TextEntity.class));
} }
@ -114,7 +123,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("s Donut ←", textEntity.getTextAfter()); assertEquals("s Donut ←", textEntity.getTextAfter());
assertEquals(searchTerm, textEntity.getValue()); assertEquals(searchTerm, textEntity.getValue());
assertEquals("Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) ", assertEquals("Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertEquals(3, textEntity.getIntersectingNodes().size()); assertEquals(3, textEntity.getIntersectingNodes().size());
assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(Paragraph.class, textEntity.getDeepestFullyContainingNode()); assertInstanceOf(Paragraph.class, textEntity.getDeepestFullyContainingNode());
@ -181,11 +190,18 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
.allEntriesInOrder()// .allEntriesInOrder()//
.filter(entry -> entry.getType().equals(NodeType.TABLE))// .filter(entry -> entry.getType().equals(NodeType.TABLE))//
.map(DocumentTree.Entry::getNode)// .map(DocumentTree.Entry::getNode)//
.findFirst().orElseThrow(); .findFirst()
.orElseThrow();
assertEquals(5, table.getNumberOfCols()); assertEquals(5, table.getNumberOfCols());
assertEquals(4, table.getNumberOfRows()); assertEquals(4, table.getNumberOfRows());
assertEquals(5, table.streamHeaders().toList().size()); assertEquals(5,
CharSequence firstHeader = table.streamHeadersForCell(1, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).findFirst().orElseThrow(); table.streamHeaders()
.toList().size());
CharSequence firstHeader = table.streamHeadersForCell(1, 1)
.map(TableCell::getTextBlock)
.map(TextBlock::getSearchText)
.findFirst()
.orElseThrow();
assertEquals("Author(s)", firstHeader.toString().stripTrailing()); assertEquals("Author(s)", firstHeader.toString().stripTrailing());
} }
@ -207,17 +223,23 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
public void assertTableStructureMetolachlor() { public void assertTableStructureMetolachlor() {
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
Table table = (Table) document.getDocumentTree() Table table = (Table) document.getDocumentTree().allEntriesInOrder()
.allEntriesInOrder() .filter(entry -> entry.getNode().getPages()
.filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22)) .stream()
.anyMatch(page -> page.getNumber() == 22))
.filter(entry -> entry.getType().equals(NodeType.TABLE)) .filter(entry -> entry.getType().equals(NodeType.TABLE))
.map(DocumentTree.Entry::getNode) .map(DocumentTree.Entry::getNode)
.findFirst() .findFirst()
.orElseThrow(); .orElseThrow();
assertEquals(5, table.getNumberOfCols()); assertEquals(5, table.getNumberOfCols());
assertEquals(14, table.getNumberOfRows()); assertEquals(14, table.getNumberOfRows());
assertEquals(10, table.streamHeaders().toList().size()); assertEquals(10,
List<String> twoHeaders = table.streamHeadersForCell(2, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).toList(); table.streamHeaders()
.toList().size());
List<String> twoHeaders = table.streamHeadersForCell(2, 1)
.map(TableCell::getTextBlock)
.map(TextBlock::getSearchText)
.toList();
assertEquals(2, twoHeaders.size()); assertEquals(2, twoHeaders.size());
assertEquals("Component of residue definition: S-Metolachlor", twoHeaders.get(0).stripTrailing()); assertEquals("Component of residue definition: S-Metolachlor", twoHeaders.get(0).stripTrailing());
assertEquals("Method type", twoHeaders.get(1).stripTrailing()); assertEquals("Method type", twoHeaders.get(1).stripTrailing());
@ -233,12 +255,13 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("except Cranberry; Vegetable, ", textEntity.getTextBefore()); assertEquals("except Cranberry; Vegetable, ", textEntity.getTextBefore());
assertEquals(", Group 9;", textEntity.getTextAfter()); assertEquals(", Group 9;", textEntity.getTextAfter());
assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ", assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ", textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertEquals(searchTerm, textEntity.getValue()); assertEquals(searchTerm, textEntity.getValue());
assertEquals(3, textEntity.getIntersectingNodes().size()); assertEquals(3, textEntity.getIntersectingNodes().size());
assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(textEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 10)); assertTrue(textEntity.getPages()
.stream()
.allMatch(pageNode -> pageNode.getNumber() == 10));
assertInstanceOf(Paragraph.class, textEntity.getDeepestFullyContainingNode()); assertInstanceOf(Paragraph.class, textEntity.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity); assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity);
@ -262,11 +285,13 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("2.6.1 Summary of ", textEntity.getTextBefore()); assertEquals("2.6.1 Summary of ", textEntity.getTextBefore());
assertEquals(" and excretion in", textEntity.getTextAfter()); assertEquals(" and excretion in", textEntity.getTextAfter());
assertEquals("2.6.1 Summary of absorption, distribution, metabolism and excretion in mammals ", assertEquals("2.6.1 Summary of absorption, distribution, metabolism and excretion in mammals ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertEquals(searchTerm, textEntity.getValue()); assertEquals(searchTerm, textEntity.getValue());
assertEquals(3, textEntity.getIntersectingNodes().size()); assertEquals(3, textEntity.getIntersectingNodes().size());
assertEquals(4, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(4, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(textEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 33)); assertTrue(textEntity.getPages()
.stream()
.allMatch(pageNode -> pageNode.getNumber() == 33));
assertInstanceOf(Headline.class, textEntity.getDeepestFullyContainingNode()); assertInstanceOf(Headline.class, textEntity.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity); assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity);
@ -285,8 +310,10 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals(searchTerm, textEntity.getValue()); assertEquals(searchTerm, textEntity.getValue());
assertEquals(4, textEntity.getIntersectingNodes().size()); assertEquals(4, textEntity.getIntersectingNodes().size());
assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ", assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertTrue(textEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54)); assertTrue(textEntity.getPages()
.stream()
.allMatch(pageNode -> pageNode.getNumber() == 54));
assertEquals(26, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(26, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(TableCell.class, textEntity.getDeepestFullyContainingNode()); assertInstanceOf(TableCell.class, textEntity.getDeepestFullyContainingNode());
@ -298,7 +325,8 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
// this might fail, if an entity with the same name exists twice in the deepest containing node // this might fail, if an entity with the same name exists twice in the deepest containing node
private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, TextEntity textEntity) { private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, TextEntity textEntity) {
List<Integer> paragraphStart = textEntity.getIntersectingNodes().stream()// List<Integer> paragraphStart = textEntity.getIntersectingNodes()
.stream()//
.map(SemanticNode::getTextBlock)// .map(SemanticNode::getTextBlock)//
.map(textBlock -> textBlock.indexOf(searchTerm, textEntity.getDeepestFullyContainingNode().getTextRange().start()))// .map(textBlock -> textBlock.indexOf(searchTerm, textEntity.getDeepestFullyContainingNode().getTextRange().start()))//
.toList(); .toList();
@ -316,14 +344,23 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
TextRange textRange = new TextRange(start, start + searchTerm.length()); TextRange textRange = new TextRange(start, start + searchTerm.length());
TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document); TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
entityCreationService.addEntityToGraph(textEntity, document); entityCreationService.addEntityToGraph(textEntity, document);
Page pageNode = document.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); Page pageNode = document.getPages()
.stream()
.filter(page -> page.getNumber() == pageNumber)
.findFirst()
.orElseThrow();
assertEquals(textEntity.getValue(), searchTerm); assertEquals(textEntity.getValue(), searchTerm);
assertTrue(pageNode.getEntities().contains(textEntity)); assertTrue(pageNode.getEntities().contains(textEntity));
assertTrue(document.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(textEntity))); assertTrue(document.getPages()
.stream()
.filter(page -> page != pageNode)
.noneMatch(page -> page.getEntities().contains(textEntity)));
assertTrue(textEntity.getPages().contains(pageNode)); assertTrue(textEntity.getPages().contains(pageNode));
assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity); assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity);
assertTrue(textEntity.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(textEntity))); assertTrue(textEntity.getIntersectingNodes()
.stream()
.allMatch(node -> node.getEntities().contains(textEntity)));
} }
} }

View File

@ -112,15 +112,15 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
@ -155,11 +155,11 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
long droolsStart = System.currentTimeMillis(); long droolsStart = System.currentTimeMillis();
List<FileAttribute> fileAttributes = entityDroolsExecutionService.executeRules(kieContainer, List<FileAttribute> fileAttributes = entityDroolsExecutionService.executeRules(kieContainer,
document, document,
dictionary, dictionary,
Collections.emptyList(), Collections.emptyList(),
new ManualRedactions(), new ManualRedactions(),
new NerEntities()); new NerEntities());
System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart); System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart);
System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);
@ -181,7 +181,8 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
float durationMillis = ((float) (System.currentTimeMillis() - start)); float durationMillis = ((float) (System.currentTimeMillis() - start));
System.out.printf("%d calls of buildTextBlock() on document took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); System.out.printf("%d calls of buildTextBlock() on document took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
Section section = document.getMainSections().get(8); Section section = document.getMainSections()
.get(8);
start = System.currentTimeMillis(); start = System.currentTimeMillis();
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
section.getTextBlock(); section.getTextBlock();
@ -245,11 +246,19 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns); System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns);
System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size()); System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size());
for (TextEntity entity : document.getEntities()) { for (TextEntity entity : document.getEntities()) {
var foundEntity = foundEntities.stream().filter(f -> f.getId().equals(entity.getId())).findFirst().get(); var foundEntity = foundEntities.stream()
.filter(f -> f.getId().equals(entity.getId()))
.findFirst()
.get();
assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange())); assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange()));
} }
assert document.getEntities().stream().mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size(); assert document.getEntities()
assert foundEntities.stream().map(TextEntity::getId).distinct().count() == document.getEntities().size(); .stream()
.mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size();
assert foundEntities.stream()
.map(TextEntity::getId)
.distinct()
.count() == document.getEntities().size();
drawAllEntities(filename, document); drawAllEntities(filename, document);
} }
@ -265,9 +274,11 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
.stream() .stream()
.filter(entityNode -> !entityNode.removed()) .filter(entityNode -> !entityNode.removed())
.filter(TextEntity::applied) .filter(TextEntity::applied)
.flatMap(entityNode -> entityNode.getPositionsOnPagePerPage().stream()) .flatMap(entityNode -> entityNode.getPositionsOnPagePerPage()
.stream())
.filter(entityPosition -> entityPosition.getPage().equals(page)) .filter(entityPosition -> entityPosition.getPage().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) .flatMap(entityPosition -> entityPosition.getRectanglePerLine()
.stream())
.toList(); .toList();
PdfVisualisationUtility.Options options = PdfVisualisationUtility.Options.builder().strokeColor(Color.BLACK).stroke(true).build(); PdfVisualisationUtility.Options options = PdfVisualisationUtility.Options.builder().strokeColor(Color.BLACK).stroke(true).build();
@ -279,9 +290,11 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
.stream() .stream()
.filter(entityNode -> !entityNode.removed()) .filter(entityNode -> !entityNode.removed())
.filter(entityNode -> !entityNode.applied()) .filter(entityNode -> !entityNode.applied())
.flatMap(entityNode -> entityNode.getPositionsOnPagePerPage().stream()) .flatMap(entityNode -> entityNode.getPositionsOnPagePerPage()
.stream())
.filter(entityPosition -> entityPosition.getPage().equals(page)) .filter(entityPosition -> entityPosition.getPage().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) .flatMap(entityPosition -> entityPosition.getRectanglePerLine()
.stream())
.toList(); .toList();
PdfVisualisationUtility.Options options = PdfVisualisationUtility.Options.builder().strokeColor(Color.BLUE).stroke(true).build(); PdfVisualisationUtility.Options options = PdfVisualisationUtility.Options.builder().strokeColor(Color.BLUE).stroke(true).build();

View File

@ -15,8 +15,17 @@ public class DocumentTableIntegrationTest extends BuildDocumentIntegrationTest {
public void testAllTableCellAccessesCorrect() { public void testAllTableCellAccessesCorrect() {
Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06"); Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06");
Page pageFive = document.getPages().stream().filter(pageNode -> pageNode.getNumber() == 5).findFirst().get(); Page pageFive = document.getPages()
Table table = pageFive.getMainBody().stream().filter(semanticNode -> semanticNode instanceof Table).map(semanticNode -> (Table) semanticNode).findFirst().get(); .stream()
.filter(pageNode -> pageNode.getNumber() == 5)
.findFirst()
.get();
Table table = pageFive.getMainBody()
.stream()
.filter(semanticNode -> semanticNode instanceof Table)
.map(semanticNode -> (Table) semanticNode)
.findFirst()
.get();
for (int row = 0; row < table.getNumberOfRows(); row++) { for (int row = 0; row < table.getNumberOfRows(); row++) {
for (int col = 0; col < table.getNumberOfCols(); col++) { for (int col = 0; col < table.getNumberOfCols(); col++) {

View File

@ -83,8 +83,8 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
PdfVisualisationUtility.drawDocumentGraph(pdDocument, document); PdfVisualisationUtility.drawDocumentGraph(pdDocument, document);
PdfVisualisationUtility.drawTextBlock(pdDocument, PdfVisualisationUtility.drawTextBlock(pdDocument,
textBlock, textBlock,
PdfVisualisationUtility.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build()); PdfVisualisationUtility.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
File outputFile = new File(tmpFileName); File outputFile = new File(tmpFileName);
pdDocument.save(outputFile); pdDocument.save(outputFile);
} }

View File

@ -27,7 +27,8 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest {
SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true); SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true);
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
List<TextEntity> entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document).toList(); List<TextEntity> entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document)
.toList();
assertEquals(2, entities.size()); assertEquals(2, entities.size());
} }

View File

@ -79,6 +79,7 @@ class DroolsSyntaxValidationServiceTest {
assertTrue(droolsSyntaxValidation.isCompiled()); assertTrue(droolsSyntaxValidation.isCompiled());
} }
@Test @Test
@SneakyThrows @SneakyThrows
void testRulesWithAddedImports() { void testRulesWithAddedImports() {
@ -96,6 +97,7 @@ class DroolsSyntaxValidationServiceTest {
assertTrue(droolsSyntaxValidation.isCompiled()); assertTrue(droolsSyntaxValidation.isCompiled());
} }
@Test @Test
@SneakyThrows @SneakyThrows
void testDocumineRules() { void testDocumineRules() {

View File

@ -26,7 +26,9 @@ public class DroolsUpToDateTest {
public void assertAllRuleFilesAreUpToDate() { public void assertAllRuleFilesAreUpToDate() {
Path droolsPath = new ClassPathResource("drools").getFile().toPath(); Path droolsPath = new ClassPathResource("drools").getFile().toPath();
Files.walk(droolsPath).filter(DroolsUpToDateTest::isEntityRuleFile).forEach(this::validateFile); Files.walk(droolsPath)
.filter(DroolsUpToDateTest::isEntityRuleFile)
.forEach(this::validateFile);
} }

View File

@ -127,15 +127,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR) .type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build())); .build()));
mockDictionaryCalls(null); mockDictionaryCalls(null);
@ -155,29 +155,39 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
String testEntityValue1 = "Desiree"; String testEntityValue1 = "Desiree";
String testEntityValue2 = "Melanie"; String testEntityValue2 = "Melanie";
EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); assertEquals(2,
assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count()); redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.count());
assertEquals(2,
redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue2))
.count());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document)
.findFirst()
.get();
String idToResize = redactionLog.getEntityLogEntry() String idToResize = redactionLog.getEntityLogEntry()
.stream() .stream()
.filter(entry -> entry.getValue().equals(testEntityValue1)) .filter(entry -> entry.getValue().equals(testEntityValue1))
.max(Comparator.comparingInt(EntityLogEntry::getStartOffset)) .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
.get() .get().getId();
.getId();
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder() manualRedactions.getResizeRedactions()
.annotationId(idToResize) .add(ManualResizeRedaction.builder()
.value(expandedEntityKeyword) .annotationId(idToResize)
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(), .value(expandedEntityKeyword)
Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build())) .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
.addToAllDossiers(false) Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
.updateDictionary(false) .addToAllDossiers(false)
.requestDate(OffsetDateTime.now()) .updateDictionary(false)
.build()); .requestDate(OffsetDateTime.now())
.build());
request.setManualRedactions(manualRedactions); request.setManualRedactions(manualRedactions);
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
@ -188,21 +198,32 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
fileOutputStream.write(annotateResponse.getDocument()); fileOutputStream.write(annotateResponse.getDocument());
} }
EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get(); EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(expandedEntityKeyword))
.findFirst()
.get();
assertEquals(idToResize, resizedEntry.getId()); assertEquals(idToResize, resizedEntry.getId());
assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
assertEquals(1, assertEquals(1,
redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count()); redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.count());
assertEquals(1,
redactionLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED))
.count());
} }
private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(), return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
(float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(), (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
(float) rectangle2D.getWidth(), (float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(), -(float) rectangle2D.getHeight(),
pageNumber); pageNumber);
} }
@ -219,10 +240,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05") .annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId") .fileId("fileId")
.legalBasis("Something") .legalBasis("Something")
.build())); .build()));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId); manualRedactionEntry.setAnnotationId(manualAddId);
@ -232,7 +253,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction"); manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
AnalyzeRequest request = uploadFileToStorage(pdfFile); AnalyzeRequest request = uploadFileToStorage(pdfFile);
request.setManualRedactions(manualRedactions); request.setManualRedactions(manualRedactions);
@ -242,11 +263,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.annotationId("675eba69b0c2917de55462c817adaa05") .annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId") .fileId("fileId")
.legalBasis("Manual Legal Basis Change") .legalBasis("Manual Legal Basis Change")
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
.build()))); .build())));
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
@ -295,7 +316,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument()); fileOutputStream.write(annotateResponse.getDocument());
} }
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();
var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny(); var optionalEntry = redactionLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findAny();
assertTrue(optionalEntry.isPresent()); assertTrue(optionalEntry.isPresent());
assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
System.out.println("duration: " + (end - start)); System.out.println("duration: " + (end - start));
@ -345,9 +369,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertFalse(redactionLog2.getEntityLogEntry() assertFalse(redactionLog2.getEntityLogEntry()
.stream() .stream()
.filter(entry -> entry.getType().equals("published_information")) .filter(entry -> entry.getType().equals("published_information"))
.anyMatch(entry -> entry.getValue().equals("Oxford University Press"))); .anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry() var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry()
.stream() .stream()
@ -381,15 +405,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
String annotationId = "testAnnotationId"; String annotationId = "testAnnotationId";
manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder() manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
.annotationId(annotationId) .annotationId(annotationId)
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
.type("manual") .type("manual")
.value("Expand to Hint Clarissas Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") .value("Expand to Hint Clarissas Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
.positions(List.of(// .positions(List.of(//
new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
.build())); .build()));
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(annotationId) .annotationId(annotationId)
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())

View File

@ -32,18 +32,31 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualResizeRedactionTest() { public void manualResizeRedactionTest() {
Document document = buildGraph("files/new/crafted document"); Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
Set<TextEntity> biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document) Set<TextEntity> biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet()); .collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); TextEntity entity = entities.stream()
TextEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); .filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
TextEntity biggerEntity = biggerEntities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(initialId) .annotationId(initialId)
.value(biggerEntity.getValue()) .value(biggerEntity.getValue())
.positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0))) .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage()
.get(0)))
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
.updateDictionary(false) .updateDictionary(false)
.build(); .build();
@ -55,8 +68,13 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty()); assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
assertEquals(biggerEntity.getPages(), entity.getPages()); assertEquals(biggerEntity.getPages(), entity.getPages());
assertEquals(biggerEntity.getValue(), entity.getValue()); assertEquals(biggerEntity.getValue(), entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); assertEquals(initialId,
assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage().get(0).getRectanglePerLine(), entity.getPositionsOnPagePerPage().get(0).getRectanglePerLine()); entity.getPositionsOnPagePerPage()
.get(0).getId());
assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage()
.get(0).getRectanglePerLine(),
entity.getPositionsOnPagePerPage()
.get(0).getRectanglePerLine());
assertTrue(entity.resized()); assertTrue(entity.resized());
} }
@ -65,11 +83,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualForceRedactionTest() { public void manualForceRedactionTest() {
Document document = buildGraph("files/new/crafted document"); Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build(); ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
doAnalysis(document, List.of(manualForceRedaction)); doAnalysis(document, List.of(manualForceRedaction));
@ -78,8 +103,12 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertFalse(entity.getIntersectingNodes().isEmpty()); assertFalse(entity.getIntersectingNodes().isEmpty());
assertEquals(1, entity.getPages().size()); assertEquals(1, entity.getPages().size());
assertEquals("David Ksenia", entity.getValue()); assertEquals("David Ksenia", entity.getValue());
assertEquals("Something", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); assertEquals("Something",
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertFalse(entity.removed()); assertFalse(entity.removed());
assertTrue(entity.hasManualChanges()); assertTrue(entity.hasManualChanges());
assertTrue(entity.applied()); assertTrue(entity.applied());
@ -90,17 +119,26 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualIDRemovalTest() { public void manualIDRemovalTest() {
Document document = buildGraph("files/new/crafted document"); Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build(); IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
doAnalysis(document, List.of(idRemoval)); doAnalysis(document, List.of(idRemoval));
assertEquals("David Ksenia", entity.getValue()); assertEquals("David Ksenia", entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertTrue(entity.ignored()); assertTrue(entity.ignored());
} }
@ -109,11 +147,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
public void manualIDRemovalButAlsoForceRedactionTest() { public void manualIDRemovalButAlsoForceRedactionTest() {
Document document = buildGraph("files/new/crafted document"); Document document = buildGraph("files/new/crafted document");
Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document)
.collect(Collectors.toUnmodifiableSet());
TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); TextEntity entity = entities.stream()
.filter(e -> e.getPages()
.stream()
.anyMatch(p -> p.getNumber() == 1))
.findFirst()
.get();
String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); String initialId = entity.getPositionsOnPagePerPage()
.get(0).getId();
IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build(); IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build(); ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
@ -123,7 +168,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
assertFalse(entity.getIntersectingNodes().isEmpty()); assertFalse(entity.getIntersectingNodes().isEmpty());
assertEquals(1, entity.getPages().size()); assertEquals(1, entity.getPages().size());
assertEquals("David Ksenia", entity.getValue()); assertEquals("David Ksenia", entity.getValue());
assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); assertEquals(initialId,
entity.getPositionsOnPagePerPage()
.get(0).getId());
assertFalse(entity.removed()); assertFalse(entity.removed());
assertFalse(entity.ignored()); assertFalse(entity.ignored());
} }
@ -131,7 +178,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) { private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) { if (rects1.stream()
.allMatch(rect1 -> rects2.stream()
.anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
return; return;
} }
// use this for nice formatting of error message // use this for nice formatting of error message
@ -143,15 +192,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
double tolerance = 1e-1; double tolerance = 1e-1;
return Math.abs(r1.getX() - r2.getX()) < tolerance &&// return Math.abs(r1.getX() - r2.getX()) < tolerance &&//
Math.abs(r1.getY() - r2.getY()) < tolerance &&// Math.abs(r1.getY() - r2.getY()) < tolerance &&//
Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&// Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
Math.abs(r1.getHeight() - r2.getHeight()) < tolerance; Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
} }
private static List<Rectangle> toAnnotationRectangles(PositionOnPage positionsOnPage) { private static List<Rectangle> toAnnotationRectangles(PositionOnPage positionsOnPage) {
return positionsOnPage.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())).toList(); return positionsOnPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber()))
.toList();
} }

View File

@ -43,7 +43,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
OffsetDateTime start = OffsetDateTime.now(); OffsetDateTime start = OffsetDateTime.now();
String reason = "whatever"; String reason = "whatever";
Document document = buildGraphNoImages("files/new/crafted document.pdf"); Document document = buildGraphNoImages("files/new/crafted document.pdf");
List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document).peek(e -> e.apply("T.0.0", reason)).toList(); List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document)
.peek(e -> e.apply("T.0.0", reason))
.toList();
assertFalse(entities.isEmpty()); assertFalse(entities.isEmpty());
TextEntity entity = entities.get(0); TextEntity entity = entities.get(0);
assertTrue(entity.active()); assertTrue(entity.active());
@ -52,7 +54,8 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.resized()); assertFalse(entity.resized());
assertFalse(entity.ignored()); assertFalse(entity.ignored());
assertEquals("n-a", entity.getMatchedRule().getLegalBasis()); assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
String annotationId = entity.getPositionsOnPagePerPage().get(0).getId(); String annotationId = entity.getPositionsOnPagePerPage()
.get(0).getId();
// remove first // remove first
IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build(); IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
@ -73,7 +76,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.ignored()); assertFalse(entity.ignored());
assertFalse(entity.removed()); assertFalse(entity.removed());
assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions()); assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); assertEquals("coolio",
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
// remove again // remove again
IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build(); IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build();
@ -93,7 +98,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertTrue(entity.ignored()); assertTrue(entity.ignored());
assertFalse(entity.applied()); assertFalse(entity.applied());
assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override", assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override",
entity.buildReasonWithManualChangeDescriptions()); entity.buildReasonWithManualChangeDescriptions());
String legalBasis = "Yeah"; String legalBasis = "Yeah";
String section = "Some random section!"; String section = "Some random section!";
@ -110,10 +115,16 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertTrue(entity.ignored()); assertTrue(entity.ignored());
assertFalse(entity.applied()); assertFalse(entity.applied());
assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed", assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed",
entity.buildReasonWithManualChangeDescriptions()); entity.buildReasonWithManualChangeDescriptions());
assertEquals(value, entity.getManualOverwrite().getValue().orElse(entity.getValue())); assertEquals(value,
assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); entity.getManualOverwrite().getValue()
assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())); .orElse(entity.getValue()));
assertEquals(legalBasis,
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
assertEquals(section,
entity.getManualOverwrite().getSection()
.orElse(entity.getDeepestFullyContainingNode().toString()));
ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder() ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder()
.type("type") .type("type")
@ -122,9 +133,13 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
.legalBasis("") .legalBasis("")
.build(); .build();
entity.getManualOverwrite().addChange(imageRecategorizationRequest); entity.getManualOverwrite().addChange(imageRecategorizationRequest);
assertTrue(entity.getManualOverwrite().getRecategorized().isPresent()); assertTrue(entity.getManualOverwrite().getRecategorized()
assertTrue(entity.getManualOverwrite().getRecategorized().get()); .isPresent());
assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type())); assertTrue(entity.getManualOverwrite().getRecategorized()
.get());
assertEquals("type",
entity.getManualOverwrite().getType()
.orElse(entity.type()));
} }
@ -134,7 +149,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
OffsetDateTime start = OffsetDateTime.now(); OffsetDateTime start = OffsetDateTime.now();
String reason = "whatever"; String reason = "whatever";
Document document = buildGraphNoImages("files/new/crafted document.pdf"); Document document = buildGraphNoImages("files/new/crafted document.pdf");
List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document).peek(e -> e.apply("T.0.0", reason)).toList(); List<TextEntity> entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document)
.peek(e -> e.apply("T.0.0", reason))
.toList();
assertFalse(entities.isEmpty()); assertFalse(entities.isEmpty());
TextEntity entity = entities.get(0); TextEntity entity = entities.get(0);
assertTrue(entity.active()); assertTrue(entity.active());
@ -143,7 +160,8 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.resized()); assertFalse(entity.resized());
assertFalse(entity.ignored()); assertFalse(entity.ignored());
assertEquals("n-a", entity.getMatchedRule().getLegalBasis()); assertEquals("n-a", entity.getMatchedRule().getLegalBasis());
String annotationId = entity.getPositionsOnPagePerPage().get(0).getId(); String annotationId = entity.getPositionsOnPagePerPage()
.get(0).getId();
// remove first // remove first
IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build(); IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
@ -164,7 +182,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
assertFalse(entity.ignored()); assertFalse(entity.ignored());
assertFalse(entity.removed()); assertFalse(entity.removed());
assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions()); assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions());
assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); assertEquals("coolio",
entity.getManualOverwrite().getLegalBasis()
.orElse(entity.getMatchedRule().getLegalBasis()));
} }

View File

@ -122,17 +122,20 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
document, .entriesToAdd(Set.of(
TEST_DOSSIER_TEMPLATE_ID); manualRedactionEntry))
.build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertEquals(1, notFoundManualEntities.size()); assertEquals(1, notFoundManualEntities.size());
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(), List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
document, document,
notFoundManualEntities, notFoundManualEntities,
new DictionaryVersion(), new DictionaryVersion(),
0L).getEntityLogEntry(); 0L).getEntityLogEntry();
assertEquals(1, redactionLogEntries.size()); assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue()); assertEquals(value, redactionLogEntries.get(0).getValue());
@ -146,7 +149,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
Document document = buildGraph("files/new/VV-919901.pdf"); Document document = buildGraph("files/new/VV-919901.pdf");
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList(); List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document)
.toList();
assertFalse(tempEntities.isEmpty()); assertFalse(tempEntities.isEmpty());
var tempEntity = tempEntities.get(0); var tempEntity = tempEntities.get(0);
List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage() List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
@ -172,21 +176,28 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
tempEntity.removeFromGraph(); tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder()
document, .entriesToAdd(Set.of(
TEST_DOSSIER_TEMPLATE_ID); manualRedactionEntry))
.build(),
document,
TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty()); assertTrue(notFoundManualEntities.isEmpty());
assertEquals(1, document.getEntities().size()); assertEquals(1, document.getEntities().size());
return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get()); return new DocumentAndEntity(document,
document.getEntities()
.stream()
.findFirst()
.get());
} }
public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
(float) rectangle2D.getWidth(), (float) rectangle2D.getWidth(),
-(float) rectangle2D.getHeight(), -(float) rectangle2D.getHeight(),
pageNumber); pageNumber);
} }

View File

@ -22,7 +22,6 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
public static final String FILE_NAME = "test-file"; public static final String FILE_NAME = "test-file";
@Test @Test
@SneakyThrows @SneakyThrows
public void testFile() { public void testFile() {
@ -41,13 +40,15 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
redactionServiceSettings.setNerServiceEnabled(false); redactionServiceSettings.setNerServiceEnabled(false);
} }
AnalyzeRequest ar = AnalyzeRequest.builder().fileId(FILE_ID).dossierId(DOSSIER_ID) AnalyzeRequest ar = AnalyzeRequest.builder()
.analysisNumber(1).dossierTemplateId(DOSSIER_TEMPLATE_ID) .fileId(FILE_ID)
.dossierId(DOSSIER_ID)
.analysisNumber(1)
.dossierTemplateId(DOSSIER_TEMPLATE_ID)
.lastProcessed(OffsetDateTime.now()) .lastProcessed(OffsetDateTime.now())
.fileAttributes(List.of()) .fileAttributes(List.of())
.build(); .build();
// try { // try {
// var redactionLog = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".redaction-log.json").getInputStream(); // var redactionLog = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".redaction-log.json").getInputStream();
// } catch (Exception e) { // } catch (Exception e) {

View File

@ -159,14 +159,19 @@ public class LiveDataIntegrationTest {
when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>()); when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());
when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(),
Colors.class)); Colors.class));
when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> { when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
String typeName = answer.getArgument(0); String typeName = answer.getArgument(0);
var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst(); var found = types.stream()
.filter(t -> t.getType().equalsIgnoreCase(typeName))
.findFirst();
if (found.isPresent()) { if (found.isPresent()) {
var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get(); var type = types.stream()
.filter(t -> t.getType().equalsIgnoreCase(typeName))
.findFirst()
.get();
type.setEntries(getEntries(typeName, type.getTypeId())); type.setEntries(getEntries(typeName, type.getTypeId()));
type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId())); type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));
@ -188,13 +193,20 @@ public class LiveDataIntegrationTest {
when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> { when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
String typeName = answer.getArgument(0); String typeName = answer.getArgument(0);
var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst(); var found = types.stream()
.filter(t -> t.getType().equalsIgnoreCase(typeName))
.findFirst();
if (found.isPresent()) { if (found.isPresent()) {
var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get(); var type = types.stream()
.filter(t -> t.getType().equalsIgnoreCase(typeName))
.findFirst()
.get();
if (deltaTypeName.equalsIgnoreCase(typeName)) { if (deltaTypeName.equalsIgnoreCase(typeName)) {
type.setEntries(values.stream().map(l -> new DictionaryEntry(0, l, version, false, type.getTypeId())).collect(Collectors.toList())); type.setEntries(values.stream()
.map(l -> new DictionaryEntry(0, l, version, false, type.getTypeId()))
.collect(Collectors.toList()));
} else { } else {
type.setEntries(new ArrayList<>()); type.setEntries(new ArrayList<>());
} }
@ -226,7 +238,9 @@ public class LiveDataIntegrationTest {
if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) { if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
List<String> lines = IOUtils.readLines(resource.getInputStream()); List<String> lines = IOUtils.readLines(resource.getInputStream());
return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList()); return lines.stream()
.map(l -> new DictionaryEntry(0, l, 0L, false, typeId))
.collect(Collectors.toList());
} }
} }
@ -242,7 +256,9 @@ public class LiveDataIntegrationTest {
if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) { if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {
List<String> lines = IOUtils.readLines(resource.getInputStream()); List<String> lines = IOUtils.readLines(resource.getInputStream());
return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList()); return lines.stream()
.map(l -> new DictionaryEntry(0, l, 0L, false, typeId))
.collect(Collectors.toList());
} }
} }

View File

@ -68,13 +68,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
Document document = buildGraphNoImages(filePath); Document document = buildGraphNoImages(filePath);
List<NerEntities.NerEntity> entityRecognitionEntities = validateAndCombine(parseNerEntities(nerEntitiesFilePath), document); List<NerEntities.NerEntity> entityRecognitionEntities = validateAndCombine(parseNerEntities(nerEntitiesFilePath), document);
assertFalse(entityRecognitionEntities.isEmpty()); assertFalse(entityRecognitionEntities.isEmpty());
assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.textRange().start() < entity.textRange().end())); assertTrue(entityRecognitionEntities.stream()
.allMatch(entity -> entity.textRange().start() < entity.textRange().end()));
ClassPathResource resource = new ClassPathResource(filePath); ClassPathResource resource = new ClassPathResource(filePath);
try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) {
Stream<NerEntities.NerEntity> unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) Stream<NerEntities.NerEntity> unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document).getNerEntityList()
.getNerEntityList()
.stream() .stream()
.filter(e -> !e.type().equals("CBI_author")); .filter(e -> !e.type().equals("CBI_author"));
List<TextEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts) List<TextEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
@ -83,12 +83,15 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
.map(Optional::get) .map(Optional::get)
.toList(); .toList();
redactionEntities.stream() redactionEntities.stream()
.collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) .collect(Collectors.groupingBy(e -> e.getPages()
.stream()
.findFirst()
.get().getNumber()))
.forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber, .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber,
getPositionsFromEntityOfType("CBI_author", entities), getPositionsFromEntityOfType("CBI_author", entities),
getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities),
getPositionsFromEntityOfType("CBI_address", entities), getPositionsFromEntityOfType("CBI_address", entities),
pdDocument)); pdDocument));
File outputFile = new File("/tmp/nerEntities.pdf"); File outputFile = new File("/tmp/nerEntities.pdf");
pdDocument.save(outputFile); pdDocument.save(outputFile);
@ -109,7 +112,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
log.info("Parsed NerEntitiesModel"); log.info("Parsed NerEntitiesModel");
NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document); NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
log.info("Validated and mapped"); log.info("Validated and mapped");
List<TextRange> nerEntityBoundaries = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList(); List<TextRange> nerEntityBoundaries = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.toList();
log.info("Combined to CBI_address"); log.info("Combined to CBI_address");
List<TextEntity> cbiAddressEntities = nerEntityBoundaries.stream() List<TextEntity> cbiAddressEntities = nerEntityBoundaries.stream()
.map(b -> entityCreationService.byTextRange(b, "CBI_address", EntityType.RECOMMENDATION, document)) .map(b -> entityCreationService.byTextRange(b, "CBI_address", EntityType.RECOMMENDATION, document))
@ -117,25 +121,28 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
.map(Optional::get) .map(Optional::get)
.toList(); .toList();
assertFalse(cbiAddressEntities.isEmpty()); assertFalse(cbiAddressEntities.isEmpty());
assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end())); assertTrue(cbiAddressEntities.stream()
.allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end()));
ClassPathResource resource = new ClassPathResource(filePath); ClassPathResource resource = new ClassPathResource(filePath);
try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) {
List<TextEntity> validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) List<TextEntity> validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document).getNerEntityList()
.getNerEntityList()
.stream() .stream()
.map(e -> entityCreationService.byTextRange(e.textRange(), e.type(), EntityType.ENTITY, document)) .map(e -> entityCreationService.byTextRange(e.textRange(), e.type(), EntityType.ENTITY, document))
.filter(Optional::isPresent) .filter(Optional::isPresent)
.map(Optional::get) .map(Optional::get)
.toList(); .toList();
Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream()) Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream())
.collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) .collect(Collectors.groupingBy(e -> e.getPages()
.stream()
.findFirst()
.get().getNumber()))
.forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber, .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber,
getPositionsFromEntityOfType("CBI_author", entities), getPositionsFromEntityOfType("CBI_author", entities),
getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities),
getPositionsFromEntityOfType("CBI_address", entities), getPositionsFromEntityOfType("CBI_address", entities),
pdDocument)); pdDocument));
File outputFile = new File("/tmp/nerEntities.pdf"); File outputFile = new File("/tmp/nerEntities.pdf");
pdDocument.save(outputFile); pdDocument.save(outputFile);
@ -147,11 +154,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document); NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList(); List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author")
.toList();
Stream<NerEntities.NerEntity> cbiAddress = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) Stream<NerEntities.NerEntity> cbiAddress = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> new NerEntities.NerEntity(document.getTextBlock().subSequence(boundary).toString(), boundary, "CBI_address")); .map(boundary -> new NerEntities.NerEntity(document.getTextBlock().subSequence(boundary).toString(), boundary, "CBI_address"));
return Stream.concat(cbiAuthors.stream(), cbiAddress).toList(); return Stream.concat(cbiAuthors.stream(), cbiAddress)
.toList();
} }
@ -167,14 +176,17 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
private List<Rectangle2D> getPositionsFromEntityOfType(String type, List<TextEntity> entities) { private List<Rectangle2D> getPositionsFromEntityOfType(String type, List<TextEntity> entities) {
return getPositionsFromEntities(entities.stream().filter(e -> e.type().equals(type))); return getPositionsFromEntities(entities.stream()
.filter(e -> e.type().equals(type)));
} }
private List<Rectangle2D> getPositionsFromEntityNotOfType(List<String> types, List<TextEntity> entities) { private List<Rectangle2D> getPositionsFromEntityNotOfType(List<String> types, List<TextEntity> entities) {
return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.type().equals(type)))); return getPositionsFromEntities(entities.stream()
.filter(e -> types.stream()
.noneMatch(type -> e.type().equals(type))));
} }
@ -188,13 +200,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, cbiAuthorRects, PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.blue).build()); PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, cbiAuthorRects, PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.blue).build());
PdfVisualisationUtility.drawRectangle2DList(pdDocument, PdfVisualisationUtility.drawRectangle2DList(pdDocument,
pageNumber, pageNumber,
addressPartsRects, addressPartsRects,
PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.MAGENTA).build()); PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.MAGENTA).build());
PdfVisualisationUtility.drawRectangle2DList(pdDocument, PdfVisualisationUtility.drawRectangle2DList(pdDocument,
pageNumber, pageNumber,
cbiAddressRects, cbiAddressRects,
PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.green).build()); PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.green).build());
} }

View File

@ -30,7 +30,13 @@ public class RegExPatternTest {
@Test @Test
public void testEmailRegEx() { public void testEmailRegEx() {
String text = "Address: Schwarzwaldalle " + "P.O.Box\n" + "CH-4002 Basel\n" + "Switzerland\n" + "Contact: Christian Warmers\n" + "Tel: +41 (61) 323 8044\n" + "christian.warmers@syngenta.com"; String text = "Address: Schwarzwaldalle "
+ "P.O.Box\n"
+ "CH-4002 Basel\n"
+ "Switzerland\n"
+ "Contact: Christian Warmers\n"
+ "Tel: +41 (61) 323 8044\n"
+ "christian.warmers@syngenta.com";
Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern.CASE_INSENSITIVE); Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern.CASE_INSENSITIVE);
@ -46,7 +52,31 @@ public class RegExPatternTest {
@Test @Test
public void testEtAlRegEx() { public void testEtAlRegEx() {
String text = "To assess the potential of S-metolachlor to cause endocrine disruption (ED) a review (Charlton 2014,\n" + "ASB2016-762) was submitted that summarises results from regulatory and open scientific literature\n" + "studies covering in vitro and in vivo studies (level 2-5 of the OECD Conceptual Framework). According to this information metolachlor increased (1.5-fold) aromatase activity in JEG-3 cells (Laville et al.\n" + "2006, ASB2010-14391) and induced weak anti-androgenic activity in the MDA-kb2 reporter cell line\n" + "with a IC50 of 9.92 µM (IC50 of positive control flutamide: 0.51 µM) (Aït-Aïssa et al. 2010, ASB2015-\n" + "9562). Data from the Tox21 high throughput screening revealed just few postive findings in assays to\n" + "identify antagonists of the androgen receptor. An isolated result of this screening showed agonistic\n" + "activity on the thyroid stimulating hormone receptor, while Dalton et al. (2003, ASB2018-2832)\n" + "demonstrated that metolachlor induced CYP2B1/2 and CYP3A1/2 but did not affect T4, T3 or TSH.\n" + "After prepubertal exposure of male Wistar rats to metolachlor (Mathias et al. 2012, ASB2016-9890) a\n" + "statistically significant increase of serum hormone concentration was observed for testosterone (at the\n" + "dose 50 mg/kg) as well as a statistically significant decrease in the age of preputial separation at a dose\n" + "of 5 and 50 mg/kg. Furthermore a statistically significant increase for estradiol at a dose of 50 mg/kg\n" + "and for FSH at a dose of 5 and 50 mg/kg and morphological alterations of the seminiferous epithelium\n" + "were observed. Relative testicular weight was not altered. A statistically significant increase of relative\n" + "weights was observed in long-term studies with rats (Tisdel et al. 1983, TOX9800328 ). This finding\n" + "was attributed to lower terminal body weight. In mice a statistically significant decrease of the weight\n" + "seminal vesicle (Tisdel et al. 1982, TOX9800327) was shown after 24 month treatment with\n" + "metolachlor. In a mouse preimplantation embryo assay from open literature metolachlor increased the\n" + "percentage of apoptosis significantly and reduced the mean number of cells per embryo significantly\n" + "while the percentage of developing blastocytes was unaltered (Grennlee et al. 2004, ASB2016-9889).\n" + "In reproduvtive toxicity studies a retarded body weight development of the pups was observed, while\n" + "survival and normal morphological and functional development were not altered. No adverse effects\n" + "on male fertility were seen, however important parameters to assess effects on female fertility like\n" + "cyclicity, ovarian follicles as well as developmental landmarks in the offspring have not been investigated."; String text = "To assess the potential of S-metolachlor to cause endocrine disruption (ED) a review (Charlton 2014,\n"
+ "ASB2016-762) was submitted that summarises results from regulatory and open scientific literature\n"
+ "studies covering in vitro and in vivo studies (level 2-5 of the OECD Conceptual Framework). According to this information metolachlor increased (1.5-fold) aromatase activity in JEG-3 cells (Laville et al.\n"
+ "2006, ASB2010-14391) and induced weak anti-androgenic activity in the MDA-kb2 reporter cell line\n"
+ "with a IC50 of 9.92 µM (IC50 of positive control flutamide: 0.51 µM) (Aït-Aïssa et al. 2010, ASB2015-\n"
+ "9562). Data from the Tox21 high throughput screening revealed just few postive findings in assays to\n"
+ "identify antagonists of the androgen receptor. An isolated result of this screening showed agonistic\n"
+ "activity on the thyroid stimulating hormone receptor, while Dalton et al. (2003, ASB2018-2832)\n"
+ "demonstrated that metolachlor induced CYP2B1/2 and CYP3A1/2 but did not affect T4, T3 or TSH.\n"
+ "After prepubertal exposure of male Wistar rats to metolachlor (Mathias et al. 2012, ASB2016-9890) a\n"
+ "statistically significant increase of serum hormone concentration was observed for testosterone (at the\n"
+ "dose 50 mg/kg) as well as a statistically significant decrease in the age of preputial separation at a dose\n"
+ "of 5 and 50 mg/kg. Furthermore a statistically significant increase for estradiol at a dose of 50 mg/kg\n"
+ "and for FSH at a dose of 5 and 50 mg/kg and morphological alterations of the seminiferous epithelium\n"
+ "were observed. Relative testicular weight was not altered. A statistically significant increase of relative\n"
+ "weights was observed in long-term studies with rats (Tisdel et al. 1983, TOX9800328 ). This finding\n"
+ "was attributed to lower terminal body weight. In mice a statistically significant decrease of the weight\n"
+ "seminal vesicle (Tisdel et al. 1982, TOX9800327) was shown after 24 month treatment with\n"
+ "metolachlor. In a mouse preimplantation embryo assay from open literature metolachlor increased the\n"
+ "percentage of apoptosis significantly and reduced the mean number of cells per embryo significantly\n"
+ "while the percentage of developing blastocytes was unaltered (Grennlee et al. 2004, ASB2016-9889).\n"
+ "In reproduvtive toxicity studies a retarded body weight development of the pups was observed, while\n"
+ "survival and normal morphological and functional development were not altered. No adverse effects\n"
+ "on male fertility were seen, however important parameters to assess effects on female fertility like\n"
+ "cyclicity, ovarian follicles as well as developmental landmarks in the offspring have not been investigated.";
Pattern p = Pattern.compile("([^\\s(]*?( \\w\\.?)?) et al\\.?"); Pattern p = Pattern.compile("([^\\s(]*?( \\w\\.?)?) et al\\.?");
@ -64,7 +94,6 @@ public class RegExPatternTest {
String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO"; String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO";
Pattern pattern = Pattern.compile("[A-ZÄÖÜ][\\wäöüéèê]{2,}( [A-ZÄÖÜ]{1,2}\\.)+"); Pattern pattern = Pattern.compile("[A-ZÄÖÜ][\\wäöüéèê]{2,}( [A-ZÄÖÜ]{1,2}\\.)+");
Matcher matcher = pattern.matcher(word); Matcher matcher = pattern.matcher(word);

Some files were not shown because too many files have changed in this diff Show More