From 9f8b1134b9d068d07e436af319f34a8d7d4d9ff3 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 1 Mar 2024 15:41:24 +0100 Subject: [PATCH] reformat --- .../redaction/v1/model/AnalyzeResponse.java | 1 + .../model/DroolsSyntaxDeprecatedWarnings.java | 1 + .../v1/model/DroolsSyntaxErrorMessage.java | 1 + .../v1/model/DroolsSyntaxValidation.java | 1 + .../v1/model/UnprocessedManualEntity.java | 1 + .../redaction/v1/server/Application.java | 3 + .../v1/server/DeprecatedElementsFinder.java | 1 + .../v1/server/RedisCachingConfiguration.java | 2 +- .../client/model/EntityRecognitionEntity.java | 1 - .../controller/RuleBuilderController.java | 1 - .../LegacyRedactionLogMergeService.java | 7 +- .../LegacyVersion0MigrationService.java | 9 +- .../redaction/v1/server/model/KieWrapper.java | 1 + .../v1/server/model/MigratedEntityLog.java | 1 + .../v1/server/model/NerEntities.java | 6 +- .../v1/server/model/component/Entity.java | 3 +- .../server/model/dictionary/Dictionary.java | 18 +- .../model/dictionary/DictionaryModel.java | 38 +- .../dictionary/SearchImplementation.java | 31 +- .../server/model/document/DocumentTree.java | 26 +- .../v1/server/model/document/TextRange.java | 19 +- .../server/model/document/entity/IEntity.java | 42 ++- .../model/document/entity/MatchedRule.java | 17 +- .../model/document/entity/TextEntity.java | 37 +- .../server/model/document/nodes/Document.java | 13 +- .../server/model/document/nodes/Headline.java | 4 +- .../v1/server/model/document/nodes/Image.java | 2 +- .../v1/server/model/document/nodes/Page.java | 6 +- .../server/model/document/nodes/Section.java | 7 +- .../document/nodes/SectionIdentifier.java | 6 +- .../model/document/nodes/SemanticNode.java | 123 ++++-- .../model/document/nodes/TableCell.java | 4 +- .../document/textblock/AtomicTextBlock.java | 39 +- .../textblock/ConcatenatedTextBlock.java | 29 +- .../model/document/textblock/TextBlock.java | 8 +- .../v1/server/model/drools/RuleClass.java | 3 +- .../model/drools/RuleFileBluePrint.java | 25 +- .../server/model/drools/RuleIdentifier.java | 4 +- .../server/queue/MessagingConfiguration.java | 1 + .../queue/RedactionMessageReceiver.java | 38 +- .../service/ComponentLogCreatorService.java | 38 +- .../service/DictionarySearchService.java | 9 +- .../v1/server/service/DictionaryService.java | 299 ++++++++------- .../service/EntityChangeLogService.java | 10 +- .../service/EntityLogCreatorService.java | 8 +- .../ManualChangesApplicationService.java | 33 +- .../NotFoundImportedEntitiesService.java | 16 +- .../service/document/ComponentComparator.java | 8 +- .../document/ComponentCreationService.java | 172 +++++---- .../service/document/DocumentGraphMapper.java | 30 +- .../service/document/EntityComparators.java | 2 + .../document/EntityCreationService.java | 66 ++-- .../document/EntityCreationUtility.java | 4 +- .../document/EntityEnrichmentService.java | 4 +- .../document/EntityFindingUtility.java | 64 +++- .../ImportedRedactionEntryService.java | 12 +- .../document/IntersectingNodeVisitor.java | 2 + .../document/ManualRedactionEntryService.java | 14 +- .../service/document/NerEntitiesAdapter.java | 56 ++- .../server/service/document/NodeVisitor.java | 1 + .../service/document/PropertiesMapper.java | 4 +- .../document/SemanticNodeComparators.java | 1 + .../ComponentDroolsExecutionService.java | 11 +- .../drools/DroolsSyntaxValidationService.java | 19 +- .../drools/KieContainerCreationService.java | 3 +- .../server/service/drools/RuleFileParser.java | 59 ++- .../storage/ObservedStorageService.java | 1 + .../storage/RedactionStorageService.java | 46 +-- .../storage/RuleManagementResources.java | 2 + .../v1/server/utils/DateConverter.java | 10 +- .../redaction/v1/server/utils/IdBuilder.java | 8 +- .../utils/RectangleTransformations.java | 26 +- .../server/utils/RedactionSearchUtility.java | 12 +- .../v1/server/utils/ResourceLoader.java | 3 +- .../v1/server/utils/SeparatorUtils.java | 8 +- .../AbstractRedactionIntegrationTest.java | 122 ++++-- .../redaction/v1/server/AnalysisTest.java | 18 +- .../v1/server/DictionaryServiceTest.java | 84 +++-- .../v1/server/DocumineFloraTest.java | 29 +- .../v1/server/RedactionAcceptanceTest.java | 43 ++- .../v1/server/RedactionIntegrationV2Test.java | 24 +- .../redaction/v1/server/RulesTest.java | 349 +++++++++++------- .../v1/server/annotate/AnnotateRequest.java | 1 - .../v1/server/annotate/AnnotationService.java | 3 +- .../document/entity/TextEntityTest.java | 11 +- ...cumentIEntityInsertionIntegrationTest.java | 87 +++-- .../DocumentPerformanceIntegrationTest.java | 57 +-- .../graph/DocumentTableIntegrationTest.java | 13 +- .../DocumentVisualizationIntegrationTest.java | 4 +- .../graph/SearchImplementationTest.java | 3 +- .../DroolsSyntaxValidationServiceTest.java | 2 + .../services/DroolsUpToDateTest.java | 4 +- .../ManualChangesEnd2EndTest.java | 130 ++++--- .../ManualChangesIntegrationTest.java | 102 +++-- .../manualchanges/ManualChangesUnitTest.java | 48 ++- .../manualchanges/PrecursorEntityTest.java | 41 +- .../AnalyseFileRealDataIntegrationTest.java | 9 +- .../realdata/LiveDataIntegrationTest.java | 32 +- .../adapter/NerEntitiesAdapterTest.java | 66 ++-- .../redaction/utils/RegExPatternTest.java | 35 +- .../redaction/utils/ResourceLoader.java | 9 +- .../redaction/v1/server/rules/Cbi11Test.java | 7 +- .../v1/server/rules/RulesIntegrationTest.java | 3 +- .../document/EntityComparatorsTest.java | 5 +- .../document/SemanticNodeComparatorsTest.java | 8 +- .../UnprocessedChangesServiceTest.java | 214 ++++++++--- .../StringMatchingPerformanceTest.java | 12 +- .../server/utils/MetricValidationUtils.java | 5 +- .../server/utils/PdfVisualisationUtility.java | 13 +- .../utility/rules/management/Main.java | 71 ++-- .../management/RuleManagementResources.java | 2 +- .../DefaultRuleIdentifiersFactory.java | 4 +- .../management/factory/RuleFileFactory.java | 41 +- .../management/factory/RuleFileParser.java | 24 +- .../migration/RuleIdentifierMigrator.java | 13 +- .../management/models/ApplicationType.java | 1 + .../rules/management/models/BasicRule.java | 2 + .../rules/management/models/RuleClass.java | 4 +- .../management/models/RuleFileBluePrint.java | 60 ++- .../management/models/RuleIdentifier.java | 9 +- .../rules/management/models/RuleType.java | 16 +- .../translation/OldRulesParser.java | 26 +- .../rules/management/utils/RuleFileIO.java | 9 +- .../management/DroolsCompilationTest.java | 8 +- .../management/RuleFileMigrationTest.java | 6 +- .../factory/RuleFileFactoryTest.java | 24 +- .../translation/OldRulesParserTest.java | 34 +- 127 files changed, 2363 insertions(+), 1225 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java index 93c983fe..ccc32f78 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java @@ -15,4 +15,5 @@ public class AnalyzeResponse { private String fileId; private List unprocessedManualEntities; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java index 787beb5f..08e0e65b 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java @@ -17,4 +17,5 @@ public class DroolsSyntaxDeprecatedWarnings { Integer line; Integer column; String message; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java index aa32db47..e6312466 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java @@ -17,4 +17,5 @@ public class DroolsSyntaxErrorMessage { Integer line; Integer column; String message; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java index ece14831..3d35c2a3 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java @@ -28,6 +28,7 @@ public class DroolsSyntaxValidation { getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build()); } + public boolean isCompiled() { return droolsSyntaxErrorMessages.isEmpty(); diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java index 3d6f92f5..dda398f7 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java @@ -24,4 +24,5 @@ public class UnprocessedManualEntity { private String section; @Builder.Default private List positions = new ArrayList<>(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index 52f9a9ed..c87b2c59 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -35,11 +35,14 @@ public class Application { SpringApplication.run(Application.class, args); } + @Bean public ObservedAspect observedAspect(ObservationRegistry observationRegistry) { + return new ObservedAspect(observationRegistry); } + @Bean public TimedAspect timedAspect(MeterRegistry registry) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java index 9383edeb..2b0e3bc5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java @@ -95,6 +95,7 @@ public class DeprecatedElementsFinder { return this.deprecatedClasses; } + private String getMethodSignature(Method method) { String methodName = method.getName(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java index df9d50b4..15a169d3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java @@ -16,7 +16,7 @@ public class RedisCachingConfiguration { public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() { return (builder) -> builder.withCacheConfiguration("documentDataCache", - RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues()); + RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java index 80975e15..05009f30 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.client.model; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 47c637fa..306dccbe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -13,7 +13,6 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { - @Override public RuleBuilderModel getRuleBuilderModel() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java index 0d141c87..48a6f1d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java @@ -92,11 +92,16 @@ public class LegacyRedactionLogMergeService { return redactionLog; } + public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) { - return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count(); + return createManualRedactionWrappers(manualRedactions).stream() + .map(ManualRedactionWrapper::getId) + .distinct() + .count(); } + private List createManualRedactionWrappers(ManualRedactions manualRedactions) { List manualRedactionWrappers = new ArrayList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java index 8b27efe7..352563aa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java @@ -21,7 +21,9 @@ public class LegacyVersion0MigrationService { public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) { List mergedEntries = new LinkedList<>(); - Map> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId)); + Map> entriesById = redactionLog.getRedactionLogEntry() + .stream() + .collect(Collectors.groupingBy(RedactionLogEntry::getId)); for (List entries : entriesById.values()) { if (entries.isEmpty()) { @@ -33,7 +35,10 @@ public class LegacyVersion0MigrationService { continue; } - List sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList(); + List sortedEntries = entries.stream() + .sorted(Comparator.comparing(entry -> entry.getChanges() + .get(0).getDateTime())) + .toList(); RedactionLogEntry initialEntry = sortedEntries.get(0); for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java index 1cacd5b1..96adca01 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java @@ -14,4 +14,5 @@ public record KieWrapper(KieContainer container, long rulesVersion) { return container != null && rulesVersion >= 0; } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java index 31a9e30d..5ef49a8d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java @@ -19,4 +19,5 @@ public class MigratedEntityLog { MigratedIds migratedIds; EntityLog entityLog; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java index 71c1c56f..c81707b0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java @@ -27,13 +27,15 @@ public class NerEntities { public boolean hasEntitiesOfType(String type) { - return nerEntityList.stream().anyMatch(nerEntity -> nerEntity.type.equals(type)); + return nerEntityList.stream() + .anyMatch(nerEntity -> nerEntity.type.equals(type)); } public Stream streamEntitiesOfType(String type) { - return nerEntityList.stream().filter(nerEntity -> nerEntity.type().equals(type)); + return nerEntityList.stream() + .filter(nerEntity -> nerEntity.type().equals(type)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java index dd7cfed4..ca4feee7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java @@ -88,7 +88,8 @@ public class Entity { .textAfter(e.getTextAfter()) .startOffset(e.getStartOffset()) .endOffset(e.getEndOffset()) - .length(Optional.ofNullable(e.getValue()).orElse("").length()) + .length(Optional.ofNullable(e.getValue()) + .orElse("").length()) .imageHasTransparency(e.isImageHasTransparency()) .isDictionaryEntry(e.isDictionaryEntry()) .isDossierDictionaryEntry(e.isDossierDictionaryEntry()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index fad8c114..703154cf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -53,7 +53,8 @@ public class Dictionary { public boolean hasLocalEntries() { - return dictionaryModels.stream().anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty()); + return dictionaryModels.stream() + .anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty()); } @@ -116,12 +117,18 @@ public class Dictionary { } localAccessMap.get(type) .getLocalEntriesWithMatchedRules() - .merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet())); + .merge(cleanedValue.trim(), + matchedRulesSet, + (set1, set2) -> Stream.concat(set1.stream(), set2.stream()) + .collect(Collectors.toSet())); if (alsoAddLastname) { String lastname = cleanedValue.split(" ")[0]; localAccessMap.get(type) .getLocalEntriesWithMatchedRules() - .merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet())); + .merge(lastname, + matchedRulesSet, + (set1, set2) -> Stream.concat(set1.stream(), set2.stream()) + .collect(Collectors.toSet())); } } @@ -153,7 +160,10 @@ public class Dictionary { } else { splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n")); } - return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList(); + return splitAuthorNames.stream() + .map(String::trim) + .filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()) + .toList(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java index 7eb35472..1214d6d3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java @@ -57,14 +57,18 @@ public class DictionaryModel implements Serializable { this.falsePositives = falsePositives; this.falseRecommendations = falseRecommendations; - this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()), - caseInsensitive); - this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()), - caseInsensitive); + this.entriesSearch = new SearchImplementation(this.entries.stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntryModel::getValue) + .collect(Collectors.toList()), caseInsensitive); + this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntryModel::getValue) + .collect(Collectors.toList()), caseInsensitive); this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .collect(Collectors.toList()), caseInsensitive); } @@ -81,8 +85,10 @@ public class DictionaryModel implements Serializable { public SearchImplementation getEntriesSearch() { if (entriesSearch == null) { - this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), - caseInsensitive); + this.entriesSearch = new SearchImplementation(this.entries.stream() + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .collect(Collectors.toList()), caseInsensitive); } return entriesSearch; } @@ -92,9 +98,9 @@ public class DictionaryModel implements Serializable { if (falsePositiveSearch == null) { this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .collect(Collectors.toList()), caseInsensitive); } return falsePositiveSearch; } @@ -104,14 +110,16 @@ public class DictionaryModel implements Serializable { if (falseRecommendationsSearch == null) { this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() - .filter(e -> !e.isDeleted()) - .map(DictionaryEntry::getValue) - .collect(Collectors.toList()), caseInsensitive); + .filter(e -> !e.isDeleted()) + .map(DictionaryEntry::getValue) + .collect(Collectors.toList()), caseInsensitive); } return falseRecommendationsSearch; } + public Set getMatchedRulesForLocalDictionaryEntry(String value) { + var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value; return localEntriesWithMatchedRules.get(cleanedValue); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java index 6d8ead3e..0dfc4c95 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java @@ -76,7 +76,9 @@ public class SearchImplementation { if (ignoreCase) { textToCheck = textToCheck.toLowerCase(Locale.ROOT); } - return this.pattern.matcher(textToCheck).results().findAny().isPresent(); + return this.pattern.matcher(textToCheck).results() + .findAny() + .isPresent(); } else { return this.trie.containsMatch(textToCheck); } @@ -89,9 +91,14 @@ public class SearchImplementation { return new ArrayList<>(); } if (this.pattern != null) { - return this.pattern.matcher(text).results().map(r -> new TextRange(r.start(), r.end())).collect(Collectors.toList()); + return this.pattern.matcher(text).results() + .map(r -> new TextRange(r.start(), r.end())) + .collect(Collectors.toList()); } else { - return this.trie.parseText(text).stream().map(r -> new TextRange(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); + return this.trie.parseText(text) + .stream() + .map(r -> new TextRange(r.getStart(), r.getEnd() + 1)) + .collect(Collectors.toList()); } } @@ -103,9 +110,14 @@ public class SearchImplementation { } CharSequence subSequence = text.subSequence(region.start(), region.end()); if (this.pattern != null) { - return this.pattern.matcher(subSequence).results().map(r -> new TextRange(r.start() + region.start(), r.end() + region.start())).collect(Collectors.toList()); + return this.pattern.matcher(subSequence).results() + .map(r -> new TextRange(r.start() + region.start(), r.end() + region.start())) + .collect(Collectors.toList()); } else { - return this.trie.parseText(subSequence).stream().map(r -> new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1)).collect(Collectors.toList()); + return this.trie.parseText(subSequence) + .stream() + .map(r -> new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1)) + .collect(Collectors.toList()); } } @@ -120,9 +132,14 @@ public class SearchImplementation { if (ignoreCase) { textToCheck = textToCheck.toLowerCase(Locale.ROOT); } - return this.pattern.matcher(textToCheck).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList()); + return this.pattern.matcher(textToCheck).results() + .map(r -> new MatchPosition(r.start(), r.end())) + .collect(Collectors.toList()); } else { - return this.trie.parseText(textToCheck).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); + return this.trie.parseText(textToCheck) + .stream() + .map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)) + .collect(Collectors.toList()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java index 64077a54..0e84de5f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java @@ -40,7 +40,10 @@ public class DocumentTree { public TextBlock buildTextBlock() { - return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + return allEntriesInOrder().map(Entry::getNode) + .filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } @@ -114,13 +117,16 @@ public class DocumentTree { public Stream childNodes(List treeId) { - return getEntryById(treeId).children.stream().map(Entry::getNode); + return getEntryById(treeId).children.stream() + .map(Entry::getNode); } public Stream childNodesOfType(List treeId, NodeType nodeType) { - return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode); + return getEntryById(treeId).children.stream() + .filter(entry -> entry.node.getType().equals(nodeType)) + .map(Entry::getNode); } @@ -199,26 +205,32 @@ public class DocumentTree { public Stream allEntriesInOrder() { - return Stream.of(root).flatMap(DocumentTree::flatten); + return Stream.of(root) + .flatMap(DocumentTree::flatten); } public Stream allSubEntriesInOrder(List parentId) { - return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten); + return getEntryById(parentId).children.stream() + .flatMap(DocumentTree::flatten); } @Override public String toString() { - return String.join("\n", allEntriesInOrder().map(Entry::toString).toList()); + return String.join("\n", + allEntriesInOrder().map(Entry::toString) + .toList()); } private static Stream flatten(Entry entry) { - return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTree::flatten)); + return Stream.concat(Stream.of(entry), + entry.children.stream() + .flatMap(DocumentTree::flatten)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java index 9ec00087..b3e76926 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java @@ -92,8 +92,13 @@ public class TextRange implements Comparable { public List split(List splitIndices) { - if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) { - throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this)); + if (splitIndices.stream() + .anyMatch(idx -> !this.contains(idx))) { + throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", + splitIndices.stream() + .filter(idx -> !this.contains(idx)) + .toList(), + this)); } List splitBoundaries = new LinkedList<>(); int previousIndex = start; @@ -113,8 +118,14 @@ public class TextRange implements Comparable { public static TextRange merge(Collection boundaries) { - int minStart = boundaries.stream().mapToInt(TextRange::start).min().orElseThrow(IllegalArgumentException::new); - int maxEnd = boundaries.stream().mapToInt(TextRange::end).max().orElseThrow(IllegalArgumentException::new); + int minStart = boundaries.stream() + .mapToInt(TextRange::start) + .min() + .orElseThrow(IllegalArgumentException::new); + int maxEnd = boundaries.stream() + .mapToInt(TextRange::end) + .max() + .orElseThrow(IllegalArgumentException::new); return new TextRange(minStart, maxEnd); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index ec6f2968..9d5bdfad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -35,14 +35,16 @@ public interface IEntity { default String value() { - return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue()); + return getManualOverwrite().getValue() + .orElse(getValue() == null ? "" : getValue()); } // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner. default boolean applied() { - return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied()); + return getManualOverwrite().getApplied() + .orElse(getMatchedRule().isApplied()); } @@ -54,19 +56,22 @@ public interface IEntity { default boolean ignored() { - return getManualOverwrite().getIgnored().orElse(getMatchedRule().isIgnored()); + return getManualOverwrite().getIgnored() + .orElse(getMatchedRule().isIgnored()); } default boolean removed() { - return getManualOverwrite().getRemoved().orElse(getMatchedRule().isRemoved()); + return getManualOverwrite().getRemoved() + .orElse(getMatchedRule().isRemoved()); } default boolean resized() { - return getManualOverwrite().getResized().orElse(false); + return getManualOverwrite().getResized() + .orElse(false); } @@ -133,12 +138,12 @@ public interface IEntity { throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); } getMatchedRuleList().add(MatchedRule.builder() - .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) - .reason(reason) - .legalBasis(legalBasis) - .applied(true) - .writeValueWithLineBreaks(true) - .build()); + .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) + .reason(reason) + .legalBasis(legalBasis) + .applied(true) + .writeValueWithLineBreaks(true) + .build()); } @@ -148,12 +153,12 @@ public interface IEntity { throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); } getMatchedRuleList().add(MatchedRule.builder() - .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) - .reason(reason) - .legalBasis(legalBasis) - .applied(true) - .references(new HashSet<>(references)) - .build()); + .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) + .reason(reason) + .legalBasis(legalBasis) + .applied(true) + .references(new HashSet<>(references)) + .build()); } @@ -207,7 +212,8 @@ public interface IEntity { default String legalBasis() { - return getManualOverwrite().getLegalBasis().orElse(getMatchedRule().getLegalBasis()); + return getManualOverwrite().getLegalBasis() + .orElse(getMatchedRule().getLegalBasis()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index e45d74f4..2e049685 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -52,7 +52,8 @@ public final class MatchedRule implements Comparable { if (!this.isApplied()) { return this; } - return MatchedRule.builder().ruleIdentifier(getRuleIdentifier()) + return MatchedRule.builder() + .ruleIdentifier(getRuleIdentifier()) .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks()) .legalBasis(this.getLegalBasis()) .reason(this.getReason()) @@ -97,7 +98,19 @@ public final class MatchedRule implements Comparable { @Override public String toString() { - return "MatchedRule[ruleIdentifier=" + ruleIdentifier + ", reason=" + reason + ", legalBasis=" + legalBasis + ", applied=" + applied + ", writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", references=" + references + ']'; + return "MatchedRule[ruleIdentifier=" + + ruleIdentifier + + ", reason=" + + reason + + ", legalBasis=" + + legalBasis + + ", applied=" + + applied + + ", writeValueWithLineBreaks=" + + writeValueWithLineBreaks + + ", references=" + + references + + ']'; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index a649889d..af4a4ac2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -67,7 +67,13 @@ public class TextEntity implements IEntity { public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { - return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build(); + return TextEntity.builder() + .id(buildId(node, textRange, type, entityType)) + .type(type) + .entityType(entityType) + .textRange(textRange) + .manualOverwrite(new ManualChangeOverwrite(entityType)) + .build(); } @@ -80,7 +86,13 @@ public class TextEntity implements IEntity { private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) { Map> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange); - return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name()); + return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), + rectanglesPerLinePerPage.values() + .stream() + .flatMap(Collection::stream) + .toList(), + type, + entityType.name()); } @@ -89,15 +101,18 @@ public class TextEntity implements IEntity { duplicateTextRanges.add(textRange); } + public boolean occursInNodeOfType(Class clazz) { - return intersectingNodes.stream().anyMatch(clazz::isInstance); + return intersectingNodes.stream() + .anyMatch(clazz::isInstance); } public boolean occursInNode(SemanticNode semanticNode) { - return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode)); + return intersectingNodes.stream() + .anyMatch(node -> node.equals(semanticNode)); } @@ -146,7 +161,10 @@ public class TextEntity implements IEntity { .min(Comparator.comparingInt(Page::getNumber)) .orElseThrow(() -> new RuntimeException("No Positions found on any page!")); - positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList(); + positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet() + .stream() + .map(entry -> buildPositionOnPage(firstPage, id, entry)) + .toList(); } return positionsOnPagePerPage; } @@ -194,7 +212,8 @@ public class TextEntity implements IEntity { public boolean matchesAnnotationId(String manualRedactionId) { - return getPositionsOnPagePerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId)); + return getPositionsOnPagePerPage().stream() + .anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId)); } @@ -224,14 +243,16 @@ public class TextEntity implements IEntity { @Override public String type() { - return getManualOverwrite().getType().orElse(type); + return getManualOverwrite().getType() + .orElse(type); } @Override public String value() { - return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value); + return getManualOverwrite().getValue() + .orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java index 4208b81f..8b9b1177 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java @@ -65,13 +65,15 @@ public class Document implements GenericSemanticNode { public List
getMainSections() { - return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList()); + return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node) + .collect(Collectors.toList()); } public Stream streamTerminalTextBlocksInOrder() { - return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock); + return streamAllNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock); } @@ -92,13 +94,16 @@ public class Document implements GenericSemanticNode { @Override public Headline getHeadline() { - return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseGet(Headline::empty); + return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node) + .findFirst() + .orElseGet(Headline::empty); } private Stream streamAllNodes() { - return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode); + return documentTree.allEntriesInOrder() + .map(DocumentTree.Entry::getNode); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java index 5c2fd280..8a5ec833 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java @@ -106,7 +106,9 @@ public class Headline implements GenericSemanticNode { public boolean hasParagraphs() { - return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent(); + return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH) + .findFirst() + .isPresent(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index 8b3b7814..acc931c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -136,7 +136,7 @@ public class Image implements GenericSemanticNode, IEntity { Map bboxImage = image.getBBox(); Map bbox = this.getBBox(); //image needs to be on the same page - if(bboxImage.get(this.page) != null) { + if (bboxImage.get(this.page) != null) { Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page)); double calculatedIntersection = intersection.getWidth() * intersection.getHeight(); double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java index f741bf62..3354372c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java @@ -45,7 +45,10 @@ public class Page { public TextBlock getMainBodyTextBlock() { - return mainBody.stream().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + return mainBody.stream() + .filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } @@ -54,4 +57,5 @@ public class Page { return String.valueOf(number); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java index afc87b9f..9c2bdac9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java @@ -53,7 +53,8 @@ public class Section implements GenericSemanticNode { public boolean hasTables() { - return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent(); + return streamAllSubNodesOfType(NodeType.TABLE).findAny() + .isPresent(); } @@ -68,7 +69,9 @@ public class Section implements GenericSemanticNode { public TextBlock getTextBlock() { if (textBlock == null) { - textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } return textBlock; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java index 2ff3a30f..a47ffe6a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java @@ -72,7 +72,11 @@ public class SectionIdentifier { } identifiers.add(Integer.parseInt(numericalIdentifier.trim())); } - return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false); + return new SectionIdentifier(Format.NUMERICAL, + identifierString, + identifiers.stream() + .toList(), + false); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java index b38b97a4..798875ec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java @@ -71,7 +71,10 @@ public interface SemanticNode { */ default Page getFirstPage() { - return getTextBlock().getPages().stream().min(Comparator.comparingInt(Page::getNumber)).orElseThrow(); + return getTextBlock().getPages() + .stream() + .min(Comparator.comparingInt(Page::getNumber)) + .orElseThrow(); } @@ -97,7 +100,8 @@ public interface SemanticNode { */ default boolean onPage(int pageNumber) { - return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber); + return getPages().stream() + .anyMatch(page -> page.getNumber() == pageNumber); } @@ -249,7 +253,9 @@ public interface SemanticNode { */ default boolean hasEntitiesOfType(String type) { - return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type)); + return getEntities().stream() + .filter(TextEntity::active) + .anyMatch(redactionEntity -> redactionEntity.type().equals(type)); } @@ -262,7 +268,10 @@ public interface SemanticNode { */ default boolean hasEntitiesOfAnyType(String... types) { - return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type))); + return getEntities().stream() + .filter(TextEntity::active) + .anyMatch(redactionEntity -> Arrays.stream(types) + .anyMatch(type -> redactionEntity.type().equals(type))); } @@ -275,7 +284,12 @@ public interface SemanticNode { */ default boolean hasEntitiesOfAllTypes(String... types) { - return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList()); + return getEntities().stream() + .filter(TextEntity::active) + .map(TextEntity::type) + .collect(Collectors.toUnmodifiableSet()) + .containsAll(Arrays.stream(types) + .toList()); } @@ -288,7 +302,10 @@ public interface SemanticNode { */ default List getEntitiesOfType(String type) { - return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList(); + return getEntities().stream() + .filter(TextEntity::active) + .filter(redactionEntity -> redactionEntity.type().equals(type)) + .toList(); } @@ -301,7 +318,10 @@ public interface SemanticNode { */ default List getEntitiesOfType(List types) { - return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList(); + return getEntities().stream() + .filter(TextEntity::active) + .filter(redactionEntity -> redactionEntity.isAnyType(types)) + .toList(); } @@ -314,7 +334,11 @@ public interface SemanticNode { */ default List getEntitiesOfType(String... types) { - return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList(); + return getEntities().stream() + .filter(TextEntity::active) + .filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types) + .toList())) + .toList(); } @@ -328,7 +352,8 @@ public interface SemanticNode { TextBlock textBlock = getTextBlock(); if (!textBlock.getAtomicTextBlocks().isEmpty()) { - return getTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage(); + return getTextBlock().getAtomicTextBlocks() + .get(0).getNumberOnPage(); } else { return -1; } @@ -357,14 +382,16 @@ public interface SemanticNode { return getTextBlock().getSearchText().contains(string); } + Set getEngines(); + default void addEngine(LayoutEngine engine) { + getEngines().add(engine); } - /** * Checks whether this SemanticNode contains all the provided Strings. * @@ -373,7 +400,8 @@ public interface SemanticNode { */ default boolean containsAllStrings(String... strings) { - return Arrays.stream(strings).allMatch(this::containsString); + return Arrays.stream(strings) + .allMatch(this::containsString); } @@ -385,7 +413,8 @@ public interface SemanticNode { */ default boolean containsAnyString(String... strings) { - return Arrays.stream(strings).anyMatch(this::containsString); + return Arrays.stream(strings) + .anyMatch(this::containsString); } @@ -397,7 +426,8 @@ public interface SemanticNode { */ default boolean containsAnyString(List strings) { - return strings.stream().anyMatch(this::containsString); + return strings.stream() + .anyMatch(this::containsString); } @@ -421,7 +451,8 @@ public interface SemanticNode { */ default boolean containsAnyStringIgnoreCase(String... strings) { - return Arrays.stream(strings).anyMatch(this::containsStringIgnoreCase); + return Arrays.stream(strings) + .anyMatch(this::containsStringIgnoreCase); } @@ -433,7 +464,8 @@ public interface SemanticNode { */ default boolean containsAllStringsIgnoreCase(String... strings) { - return Arrays.stream(strings).allMatch(this::containsStringIgnoreCase); + return Arrays.stream(strings) + .allMatch(this::containsStringIgnoreCase); } @@ -445,7 +477,9 @@ public interface SemanticNode { */ default boolean containsWord(String word) { - return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word)); + return getTextBlock().getWords() + .stream() + .anyMatch(s -> s.equals(word)); } @@ -457,7 +491,10 @@ public interface SemanticNode { */ default boolean containsWordIgnoreCase(String word) { - return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH))); + return getTextBlock().getWords() + .stream() + .map(String::toLowerCase) + .anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH))); } @@ -469,7 +506,10 @@ public interface SemanticNode { */ default boolean containsAnyWord(String... words) { - return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals)); + return Arrays.stream(words) + .anyMatch(word -> getTextBlock().getWords() + .stream() + .anyMatch(word::equals)); } @@ -481,7 +521,12 @@ public interface SemanticNode { */ default boolean containsAnyWordIgnoreCase(String... words) { - return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals)); + return Arrays.stream(words) + .map(String::toLowerCase) + .anyMatch(word -> getTextBlock().getWords() + .stream() + .map(String::toLowerCase) + .anyMatch(word::equals)); } @@ -493,7 +538,10 @@ public interface SemanticNode { */ default boolean containsAllWords(String... words) { - return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals)); + return Arrays.stream(words) + .allMatch(word -> getTextBlock().getWords() + .stream() + .anyMatch(word::equals)); } @@ -505,7 +553,12 @@ public interface SemanticNode { */ default boolean containsAllWordsIgnoreCase(String... words) { - return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals)); + return Arrays.stream(words) + .map(String::toLowerCase) + .allMatch(word -> getTextBlock().getWords() + .stream() + .map(String::toLowerCase) + .anyMatch(word::equals)); } @@ -545,7 +598,11 @@ public interface SemanticNode { */ default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) { - return getBBox().entrySet().stream().filter(entry -> entry.getKey().getNumber() == pageNumber).map(Map.Entry::getValue).anyMatch(rect -> rect.intersects(x, y, w, h)); + return getBBox().entrySet() + .stream() + .filter(entry -> entry.getKey().getNumber() == pageNumber) + .map(Map.Entry::getValue) + .anyMatch(rect -> rect.intersects(x, y, w, h)); } @@ -598,7 +655,8 @@ public interface SemanticNode { */ default Stream streamAllSubNodes() { - return getDocumentTree().allSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode); + return getDocumentTree().allSubEntriesInOrder(getTreeId()) + .map(DocumentTree.Entry::getNode); } @@ -609,7 +667,9 @@ public interface SemanticNode { */ default Stream streamAllSubNodesOfType(NodeType nodeType) { - return getDocumentTree().allSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode); + return getDocumentTree().allSubEntriesInOrder(getTreeId()) + .filter(entry -> entry.getType().equals(nodeType)) + .map(DocumentTree.Entry::getNode); } @@ -648,7 +708,8 @@ public interface SemanticNode { if (isLeaf()) { return getTextBlock().getPositionsPerPage(textRange); } - Optional containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst(); + Optional containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)) + .findFirst(); if (containingChildNode.isEmpty()) { return getTextBlock().getPositionsPerPage(textRange); } @@ -698,8 +759,12 @@ public interface SemanticNode { private Map getBBoxFromChildren() { Map bBoxPerPage = new HashMap<>(); - List> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList(); - Set pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet()); + List> childrenBBoxes = streamChildren().map(SemanticNode::getBBox) + .toList(); + Set pages = childrenBBoxes.stream() + .flatMap(map -> map.keySet() + .stream()) + .collect(Collectors.toSet()); for (Page page : pages) { Rectangle2D bBoxOnPage = childrenBBoxes.stream() .filter(childBboxPerPage -> childBboxPerPage.containsKey(page)) @@ -717,7 +782,9 @@ public interface SemanticNode { private Map getBBoxFromLeafTextBlock() { Map bBoxPerPage = new HashMap<>(); - Map> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage)); + Map> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks() + .stream() + .collect(Collectors.groupingBy(AtomicTextBlock::getPage)); atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks))); return bBoxPerPage; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java index 67026fff..0a1d3cf8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java @@ -79,7 +79,9 @@ public class TableCell implements GenericSemanticNode { } if (textBlock == null) { - textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } return textBlock; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java index e56c5f8a..3b6a1102 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java @@ -61,6 +61,7 @@ public class AtomicTextBlock implements TextBlock { return lineBreaks.size() + 1; } + public static AtomicTextBlock empty(Long textBlockIdx, int stringOffset, Page page, int numberOnPage, SemanticNode parent) { return AtomicTextBlock.builder() @@ -77,10 +78,7 @@ public class AtomicTextBlock implements TextBlock { } - public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData atomicTextBlockData, - DocumentPositionData atomicPositionBlockData, - SemanticNode parent, - Page page) { + public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData atomicTextBlockData, DocumentPositionData atomicPositionBlockData, SemanticNode parent, Page page) { return AtomicTextBlock.builder() .id(atomicTextBlockData.getId()) @@ -88,8 +86,10 @@ public class AtomicTextBlock implements TextBlock { .page(page) .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd())) .searchText(atomicTextBlockData.getSearchText()) - .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList()) - .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList()) + .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed() + .toList()) + .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed() + .toList()) .positions(toRectangle2DList(atomicPositionBlockData.getPositions())) .parent(parent) .build(); @@ -98,7 +98,9 @@ public class AtomicTextBlock implements TextBlock { private static List toRectangle2DList(float[][] positions) { - return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList(); + return Arrays.stream(positions) + .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])) + .toList(); } @@ -118,6 +120,7 @@ public class AtomicTextBlock implements TextBlock { return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start()); } + public List getWords() { if (words == null) { @@ -144,9 +147,9 @@ public class AtomicTextBlock implements TextBlock { public int getNextLinebreak(int fromIndex) { return lineBreaks.stream()// - .filter(linebreak -> linebreak > fromIndex - textRange.start()) // - .findFirst() // - .orElse(searchText.length()) + textRange.start(); + .filter(linebreak -> linebreak > fromIndex - textRange.start()) // + .findFirst() // + .orElse(searchText.length()) + textRange.start(); } @@ -154,9 +157,9 @@ public class AtomicTextBlock implements TextBlock { public int getPreviousLinebreak(int fromIndex) { return lineBreaks.stream()// - .filter(linebreak -> linebreak <= fromIndex - textRange.start())// - .reduce((a, b) -> b)// - .orElse(0) + textRange.start(); + .filter(linebreak -> linebreak <= fromIndex - textRange.start())// + .reduce((a, b) -> b)// + .orElse(0) + textRange.start(); } @@ -209,7 +212,10 @@ public class AtomicTextBlock implements TextBlock { return ""; } - Set lbInBoundary = lineBreaks.stream().map(i -> i + textRange.start()).filter(textRange::contains).collect(Collectors.toSet()); + Set lbInBoundary = lineBreaks.stream() + .map(i -> i + textRange.start()) + .filter(textRange::contains) + .collect(Collectors.toSet()); if (textRange.end() == getTextRange().end()) { lbInBoundary.add(getTextRange().end()); } @@ -235,7 +241,10 @@ public class AtomicTextBlock implements TextBlock { private List getAllLineBreaksInBoundary(TextRange textRange) { - return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList(); + return getLineBreaks().stream() + .map(linebreak -> linebreak + this.textRange.start()) + .filter(textRange::contains) + .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java index 9f372ed4..b69a2590 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java @@ -44,7 +44,8 @@ public class ConcatenatedTextBlock implements TextBlock { this.atomicTextBlocks.add(firstTextBlock); textRange = new TextRange(firstTextBlock.getTextRange().start(), firstTextBlock.getTextRange().end()); - atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat); + atomicTextBlocks.subList(1, atomicTextBlocks.size()) + .forEach(this::concat); } @@ -65,7 +66,10 @@ public class ConcatenatedTextBlock implements TextBlock { private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) { - return atomicTextBlocks.stream().filter(textBlock -> textBlock.getTextRange().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new); + return atomicTextBlocks.stream() + .filter(textBlock -> textBlock.getTextRange().contains(stringIdx)) + .findAny() + .orElseThrow(IndexOutOfBoundsException::new); } @@ -99,14 +103,18 @@ public class ConcatenatedTextBlock implements TextBlock { @Override public List getWords() { - return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList(); + return atomicTextBlocks.stream() + .map(AtomicTextBlock::getWords) + .flatMap(Collection::stream) + .toList(); } @Override public int numberOfLines() { - return atomicTextBlocks.stream().mapToInt(AtomicTextBlock::numberOfLines).sum(); + return atomicTextBlocks.stream() + .mapToInt(AtomicTextBlock::numberOfLines).sum(); } @@ -127,7 +135,10 @@ public class ConcatenatedTextBlock implements TextBlock { @Override public List getLineBreaks() { - return getAtomicTextBlocks().stream().flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks().stream()).toList(); + return getAtomicTextBlocks().stream() + .flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks() + .stream()) + .toList(); } @@ -202,7 +213,8 @@ public class ConcatenatedTextBlock implements TextBlock { AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1); rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, - lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end()))); + lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(), + stringTextRange.end()))); return rectanglesPerLinePerPage; } @@ -239,7 +251,10 @@ public class ConcatenatedTextBlock implements TextBlock { private Map> mergeEntityPositionsWithSamePageNode(Map> map1, Map> map2) { Map> mergedMap = new HashMap<>(map1); - map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (l1, l2) -> Stream.concat(l1.stream(), l2.stream()).toList())); + map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, + rectangles, + (l1, l2) -> Stream.concat(l1.stream(), l2.stream()) + .toList())); return mergedMap; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java index 57303ee1..89c45693 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java @@ -18,8 +18,10 @@ public interface TextBlock extends CharSequence { String getSearchText(); + List getWords(); + List getAtomicTextBlocks(); @@ -35,7 +37,6 @@ public interface TextBlock extends CharSequence { TextRange getLineTextRange(int lineNumber); - List getLineBreaks(); @@ -71,6 +72,7 @@ public interface TextBlock extends CharSequence { return RectangleTransformations.rectangle2DBBox(getLinePositions(lineNumber)); } + default String searchTextWithLineBreaks() { return subSequenceWithLineBreaks(getTextRange()); @@ -85,7 +87,9 @@ public interface TextBlock extends CharSequence { default Set getPages() { - return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet()); + return getAtomicTextBlocks().stream() + .map(AtomicTextBlock::getPage) + .collect(Collectors.toUnmodifiableSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java index 5b75502a..04983fc4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java @@ -9,7 +9,8 @@ public record RuleClass(RuleType ruleType, List ruleUnits) { public Optional findRuleUnitByInteger(Integer unit) { return ruleUnits.stream() - .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst(); + .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)) + .findFirst(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java index 8bbfbc43..5f98935c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java @@ -37,13 +37,17 @@ public final class RuleFileBluePrint { public Optional findRuleClassByType(RuleType ruleType) { - return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst(); + return ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .findFirst(); } public Set getImportSplitByKeyword() { - return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet()); + return Arrays.stream(imports.replaceAll("\n", "").split("import")) + .map(String::trim) + .collect(Collectors.toSet()); } @@ -53,11 +57,15 @@ public final class RuleFileBluePrint { return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits) .orElse(Collections.emptyList()) .stream() - .flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) + .flatMap(ruleUnit -> ruleUnit.rules() + .stream() + .filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) .toList(); } return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit())) - .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) + .map(ruleUnit -> ruleUnit.rules() + .stream() + .filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) .orElse(Stream.empty()) .toList(); } @@ -65,13 +73,18 @@ public final class RuleFileBluePrint { public List getAllRuleIdentifiers() { - return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList()); + return streamAllRules().map(BasicRule::getIdentifier) + .collect(Collectors.toList()); } public Stream streamAllRules() { - return getRuleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream); + return getRuleClasses().stream() + .map(RuleClass::ruleUnits) + .flatMap(Collection::stream) + .map(RuleUnit::rules) + .flatMap(Collection::stream); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java index 2d09c566..63a907d7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java @@ -42,8 +42,8 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { public boolean matches(RuleIdentifier ruleIdentifier) { return ruleIdentifier.type().equals(this.type()) && // - (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // - (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); + (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // + (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java index 3dc70931..1c5a3f9d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java @@ -70,6 +70,7 @@ public class MessagingConfiguration { .build(); } + @Bean public Queue redactionAnalysisResponseQueue() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index cea10030..50f01015 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -51,14 +51,14 @@ public class RedactionMessageReceiver { // This prevents from endless retries oom errors. if (message.getMessageProperties().isRedelivered()) { var errorMessage = format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", - analyzeRequest.getDossierId(), - analyzeRequest.getFileId()); + analyzeRequest.getDossierId(), + analyzeRequest.getFileId()); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - new FileErrorInfo(errorMessage, - priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, - "redaction-service", - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); + analyzeRequest.getFileId(), + new FileErrorInfo(errorMessage, + priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, + "redaction-service", + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); throw new AmqpRejectAndDontRequeueException(errorMessage); } @@ -84,9 +84,9 @@ public class RedactionMessageReceiver { log.debug(analyzeRequest.getManualRedactions().toString()); result = analyzeService.analyze(analyzeRequest); log.info("Successfully analyzed dossier {} file {} took: {} s", - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - format("%.2f", result.getDuration() / 1000.0)); + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + format("%.2f", result.getDuration() / 1000.0)); log.info("----------------------------------------------------------------------------------"); break; @@ -96,9 +96,9 @@ public class RedactionMessageReceiver { log.debug(analyzeRequest.getManualRedactions().toString()); result = analyzeService.reanalyze(analyzeRequest); log.info("Successfully reanalyzed dossier {} file {} took: {} s", - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - format("%.2f", result.getDuration() / 1000.0)); + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + format("%.2f", result.getDuration() / 1000.0)); log.info("----------------------------------------------------------------------------------"); break; case SURROUNDING_TEXT_ANALYSIS: @@ -106,9 +106,7 @@ public class RedactionMessageReceiver { log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.debug(analyzeRequest.getManualRedactions().toString()); unprocessedChangesService.analyseSurroundingText(analyzeRequest); - log.info("Successful Surrounding Text Analysis dossier {} file {} ", - analyzeRequest.getDossierId(), - analyzeRequest.getFileId()); + log.info("Successful Surrounding Text Analysis dossier {} file {} ", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); log.info("-------------------------------------------------------------------------------------------------"); shouldRespond = false; break; @@ -137,8 +135,8 @@ public class RedactionMessageReceiver { log.warn("Failed to process analyze request: {}", analyzeRequest, e); var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); + analyzeRequest.getFileId(), + new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); } @@ -153,8 +151,8 @@ public class RedactionMessageReceiver { timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp); fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp)); + analyzeRequest.getFileId(), + new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp)); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java index cce55740..1fbdc90e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -23,13 +23,15 @@ public class ComponentLogCreatorService { public ComponentLog buildComponentLog(int analysisNumber, List components, long componentRulesVersion) { Map> map = new HashMap<>(); - components.stream().sorted(ComponentComparator.first()).forEach(component -> { - ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component); - map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue); - }); - List componentLogComponents = map - .entrySet() - .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue())) + components.stream() + .sorted(ComponentComparator.first()) + .forEach(component -> { + ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component); + map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue); + }); + List componentLogComponents = map.entrySet() + .stream() + .map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue())) .toList(); return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents); } @@ -38,24 +40,36 @@ public class ComponentLogCreatorService { private ComponentLogEntryValue buildComponentLogEntry(Component component) { return ComponentLogEntryValue.builder() - .value(component.getValue()).originalValue(component.getValue()) + .value(component.getValue()) + .originalValue(component.getValue()) .componentRuleId(component.getMatchedRule().toString()) .valueDescription(component.getValueDescription()) - .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.first()).toList())) + .componentLogEntityReferences(toComponentEntityReferences(component.getReferences() + .stream() + .sorted(EntityComparators.first()) + .toList())) .build(); } private List toComponentEntityReferences(List references) { - return references.stream().map(this::toComponentEntityReference).toList(); + return references.stream() + .map(this::toComponentEntityReference) + .toList(); } private ComponentLogEntityReference toComponentEntityReference(Entity entity) { - return ComponentLogEntityReference.builder().id(entity.getId()) - .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule()) + return ComponentLogEntityReference.builder() + .id(entity.getId()) + .page(entity.getPositions() + .stream() + .findFirst() + .map(Position::getPageNumber) + .orElse(0)) + .entityRuleId(entity.getMatchedRule()) .type(entity.getType()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 7766d5f4..136b26f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -56,10 +56,11 @@ public class DictionarySearchService { searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) .stream() .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) - .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> { - entity.setDictionaryEntry(true); - entity.setDossierDictionaryEntry(isDossierDictionaryEntry); - })); + .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)) + .ifPresent(entity -> { + entity.setDictionaryEntry(true); + entity.setDossierDictionaryEntry(isDossierDictionaryEntry); + })); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index e459ba22..e91e1d26 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -106,41 +106,47 @@ public class DictionaryService { List dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary(); dictionaryModels.forEach(dictionaryModel -> { - dictionaryModel.getEntries().forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); + dictionaryModel.getEntries() + .forEach(dictionaryEntry -> { + if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { + newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); + dictionaryModel.getFalsePositives() + .forEach(dictionaryEntry -> { + if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { + newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); + dictionaryModel.getFalseRecommendations() + .forEach(dictionaryEntry -> { + if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { + newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); }); if (dossierDictionaryExists(dossierId)) { dictionaryModels = getDossierDictionary(dossierId).getDictionary(); dictionaryModels.forEach(dictionaryModel -> { - dictionaryModel.getEntries().forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); - dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> { - if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { - newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); - } - }); + dictionaryModel.getEntries() + .forEach(dictionaryEntry -> { + if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { + newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); + dictionaryModel.getFalsePositives() + .forEach(dictionaryEntry -> { + if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { + newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); + dictionaryModel.getFalseRecommendations() + .forEach(dictionaryEntry -> { + if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { + newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive())); + } + }); }); } @@ -155,84 +161,120 @@ public class DictionaryService { DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation(); var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId, - true); + true); if (CollectionUtils.isNotEmpty(typeResponse)) { - List dictionary = typeResponse.stream().map(t -> { + List dictionary = typeResponse.stream() + .map(t -> { - Optional optionalOldModel; - if (dossierId == null) { - var representation = getDossierTemplateDictionary(dossierTemplateId); - optionalOldModel = representation != null ? representation.getDictionary() - .stream() - .filter(f -> f.getType().equals(t.getType())) - .findAny() : Optional.empty(); - } else { - var representation = getDossierDictionary(dossierId); - optionalOldModel = representation != null ? representation.getDictionary() - .stream() - .filter(f -> f.getType().equals(t.getType())) - .findAny() : Optional.empty(); - } + Optional optionalOldModel; + if (dossierId == null) { + var representation = getDossierTemplateDictionary(dossierTemplateId); + optionalOldModel = representation != null ? representation.getDictionary() + .stream() + .filter(f -> f.getType().equals(t.getType())) + .findAny() : Optional.empty(); + } else { + var representation = getDossierDictionary(dossierId); + optionalOldModel = representation != null ? representation.getDictionary() + .stream() + .filter(f -> f.getType().equals(t.getType())) + .findAny() : Optional.empty(); + } - Set entries = new HashSet<>(); - Set falsePositives = new HashSet<>(); - Set falseRecommendations = new HashSet<>(); + Set entries = new HashSet<>(); + Set falsePositives = new HashSet<>(); + Set falseRecommendations = new HashSet<>(); - DictionaryEntries newEntries = getEntries(t.getId(), currentVersion); + DictionaryEntries newEntries = getEntries(t.getId(), currentVersion); - var newValues = newEntries.getEntries().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet()); - var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet()); - var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet()); - - optionalOldModel.ifPresent(oldDictionaryModel -> { - - }); - if (optionalOldModel.isPresent()) { - var oldModel = optionalOldModel.get(); - if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) { - // add old entries from existing DictionaryModel but exclude lower case representation - entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())).toList()); - falsePositives.addAll(oldModel.getFalsePositives() + var newValues = newEntries.getEntries() .stream() - .filter(f -> !newFalsePositivesValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())) - .toList()); - falseRecommendations.addAll(oldModel.getFalseRecommendations() + .map(DictionaryEntry::getValue) + .collect(Collectors.toSet()); + var newFalsePositivesValues = newEntries.getFalsePositives() .stream() - .filter(f -> !newFalseRecommendationsValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())) - .toList()); - } else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) { - // add old entries from existing DictionaryModel but exclude upper case representation - entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList()); - falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList()); - falseRecommendations.addAll(oldModel.getFalseRecommendations() + .map(DictionaryEntry::getValue) + .collect(Collectors.toSet()); + var newFalseRecommendationsValues = newEntries.getFalseRecommendations() .stream() - .filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT))) - .toList()); + .map(DictionaryEntry::getValue) + .collect(Collectors.toSet()); - } else { - // add old entries from existing DictionaryModel - entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList()); - falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue())).toList()); - falseRecommendations.addAll(oldModel.getFalseRecommendations().stream().filter(f -> !newFalseRecommendationsValues.contains(f.getValue())).toList()); - } - } + optionalOldModel.ifPresent(oldDictionaryModel -> { - // Add Increments - entries.addAll(newEntries.getEntries()); - falsePositives.addAll(newEntries.getFalsePositives()); - falseRecommendations.addAll(newEntries.getFalseRecommendations()); + }); + if (optionalOldModel.isPresent()) { + var oldModel = optionalOldModel.get(); + if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) { + // add old entries from existing DictionaryModel but exclude lower case representation + entries.addAll(oldModel.getEntries() + .stream() + .filter(f -> !newValues.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .toList().contains(f.getValue())) + .toList()); + falsePositives.addAll(oldModel.getFalsePositives() + .stream() + .filter(f -> !newFalsePositivesValues.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .toList().contains(f.getValue())) + .toList()); + falseRecommendations.addAll(oldModel.getFalseRecommendations() + .stream() + .filter(f -> !newFalseRecommendationsValues.stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .toList().contains(f.getValue())) + .toList()); + } else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) { + // add old entries from existing DictionaryModel but exclude upper case representation + entries.addAll(oldModel.getEntries() + .stream() + .filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT))) + .toList()); + falsePositives.addAll(oldModel.getFalsePositives() + .stream() + .filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT))) + .toList()); + falseRecommendations.addAll(oldModel.getFalseRecommendations() + .stream() + .filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT))) + .toList()); - return new DictionaryModel(t.getType(), - t.getRank(), - convertColor(t.getHexColor()), - t.isCaseInsensitive(), - t.isHint(), - entries, - falsePositives, - falseRecommendations, - dossierId != null); - }).sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()); + } else { + // add old entries from existing DictionaryModel + entries.addAll(oldModel.getEntries() + .stream() + .filter(f -> !newValues.contains(f.getValue())) + .toList()); + falsePositives.addAll(oldModel.getFalsePositives() + .stream() + .filter(f -> !newFalsePositivesValues.contains(f.getValue())) + .toList()); + falseRecommendations.addAll(oldModel.getFalseRecommendations() + .stream() + .filter(f -> !newFalseRecommendationsValues.contains(f.getValue())) + .toList()); + } + } + + // Add Increments + entries.addAll(newEntries.getEntries()); + falsePositives.addAll(newEntries.getFalsePositives()); + falseRecommendations.addAll(newEntries.getFalseRecommendations()); + + return new DictionaryModel(t.getType(), + t.getRank(), + convertColor(t.getHexColor()), + t.isCaseInsensitive(), + t.isHint(), + entries, + falsePositives, + falseRecommendations, + dossierId != null); + }) + .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) + .collect(Collectors.toList()); dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm)); @@ -264,17 +306,17 @@ public class DictionaryService { var type = dictionaryClient.getDictionaryForType(typeId, fromVersion); Set entries = type.getEntries() != null ? new HashSet<>(type.getEntries() - .stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet())) : new HashSet<>(); Set falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries() - .stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet())) : new HashSet<>(); Set falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries() - .stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet())) : new HashSet<>(); if (type.isCaseInsensitive()) { entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); @@ -282,10 +324,10 @@ public class DictionaryService { falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); } log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}", - entries.size(), - falsePositives.size(), - falseRecommendations.size(), - typeId); + entries.size(), + falsePositives.size(), + falseRecommendations.size(), + typeId); return new DictionaryEntries(entries, falsePositives, falseRecommendations); } @@ -300,7 +342,8 @@ public class DictionaryService { @SneakyThrows public float[] getColor(String type, String dossierTemplateId) { - DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type); + DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() + .get(type); if (model != null) { return model.getColor(); } @@ -311,7 +354,8 @@ public class DictionaryService { @SneakyThrows public boolean isHint(String type, String dossierTemplateId) { - DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type); + DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap() + .get(type); if (model != null) { return model.isHint(); } @@ -335,15 +379,20 @@ public class DictionaryService { var dossierRepresentation = getDossierDictionary(dossierId); var dossierDictionaries = dossierRepresentation.getDictionary(); mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), - convertDictionaryModel(dossierDictionaries))); + convertDictionaryModel(dossierDictionaries))); dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); } else { mergedDictionaries = new ArrayList<>(); dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); } - return new Dictionary(mergedDictionaries.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), - DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); + return new Dictionary(mergedDictionaries.stream() + .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()) + .collect(Collectors.toList()), + DictionaryVersion.builder() + .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()) + .dossierVersion(dossierDictionaryVersion) + .build()); } @@ -371,14 +420,16 @@ public class DictionaryService { @SneakyThrows private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) { - return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId); + return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate() + .get(dossierTemplateId); } @SneakyThrows private DictionaryRepresentation getDossierDictionary(String dossierId) { - return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId); + return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier() + .get(dossierId); } @@ -421,14 +472,14 @@ public class DictionaryService { return commonsDictionaries.stream() .map(cd -> new DictionaryModel(cd.getType(), - cd.getRank(), - cd.getColor(), - cd.isCaseInsensitive(), - cd.isHint(), - cd.getEntries(), - cd.getFalsePositives(), - cd.getFalseRecommendations(), - cd.isDossierDictionary())) + cd.getRank(), + cd.getColor(), + cd.isCaseInsensitive(), + cd.isHint(), + cd.getEntries(), + cd.getFalsePositives(), + cd.getFalseRecommendations(), + cd.isDossierDictionary())) .collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index a6bc570b..793ee18c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j; public class EntityChangeLogService { @Timed("redactmanager_computeChanges") - public boolean computeChanges(List previousEntityLogEntries, List newEntityLogEntries, ManualRedactions manualRedactions, int analysisNumber) { + public boolean computeChanges(List previousEntityLogEntries, List newEntityLogEntries, int analysisNumber) { var now = OffsetDateTime.now(); if (previousEntityLogEntries.isEmpty()) { @@ -58,16 +58,12 @@ public class EntityChangeLogService { entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now)); } } - addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now); + addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now); return hasChanges; } - private void addRemovedEntriesAsRemoved(List previousEntityLogEntries, - List newEntityLogEntries, - ManualRedactions manualRedactions, - int analysisNumber, - OffsetDateTime now) { + private void addRemovedEntriesAsRemoved(List previousEntityLogEntries, List newEntityLogEntries, int analysisNumber, OffsetDateTime now) { Set existingIds = newEntityLogEntries.stream() .map(EntityLogEntry::getId) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 458d3c5b..5ae1f325 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -70,7 +70,7 @@ public class EntityLogCreatorService { List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber()); + entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); return new EntityLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), @@ -128,10 +128,8 @@ public class EntityLogCreatorService { .collect(Collectors.toList()); previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); - boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, - newEntityLogEntries, - analyzeRequest.getManualRedactions(), - analyzeRequest.getAnalysisNumber()); + boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); + previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index d709f211..67ab7b56 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -74,9 +74,9 @@ public class ManualChangesApplicationService { .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions() - .stream() - .map(ManualChangesApplicationService::toRectangle2D) - .collect(Collectors.toList())); + .stream() + .map(ManualChangesApplicationService::toRectangle2D) + .collect(Collectors.toList())); entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction); @@ -90,11 +90,17 @@ public class ManualChangesApplicationService { if (closestEntity.isPresent()) { copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get()); - possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + possibleEntities.values() + .stream() + .flatMap(Collection::stream) + .forEach(TextEntity::removeFromGraph); return; } - possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + possibleEntities.values() + .stream() + .flatMap(Collection::stream) + .forEach(TextEntity::removeFromGraph); if (node.hasParent()) { node = node.getParent(); @@ -110,14 +116,18 @@ public class ManualChangesApplicationService { Set currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes()); Set newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes()); - Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized)); - Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized)); + Sets.difference(currentIntersectingNodes, newIntersectingNodes) + .forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized)); + Sets.difference(newIntersectingNodes, currentIntersectingNodes) + .forEach(addedNode -> addedNode.getEntities().add(entityToBeResized)); Set currentIntersectingPages = new HashSet<>(entityToBeResized.getPages()); Set newIntersectingPages = new HashSet<>(closestEntity.getPages()); - Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized)); - Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized)); + Sets.difference(currentIntersectingPages, newIntersectingPages) + .forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized)); + Sets.difference(newIntersectingPages, currentIntersectingPages) + .forEach(addedPage -> addedPage.getEntities().add(entityToBeResized)); entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes)); @@ -135,7 +145,10 @@ public class ManualChangesApplicationService { if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) { return; } - var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList()); + var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions() + .stream() + .map(ManualChangesApplicationService::toRectangle2D) + .toList()); image.setPosition(bBox); image.getManualOverwrite().addChange(manualResizeRedaction); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java index 159d7c6b..04e687e6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java @@ -53,10 +53,13 @@ public class NotFoundImportedEntitiesService { if (!notFoundEntities.isEmpty()) { // imported redactions present, intersections must be added with merged imported redactions Map> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities); - entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> { - redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); - addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber); - }); + entityLog.getEntityLogEntry() + .stream() + .filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)) + .forEach(redactionLogEntry -> { + redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); + addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber); + }); } } @@ -70,7 +73,10 @@ public class NotFoundImportedEntitiesService { .map(RectangleWithPage::pageNumber) .collect(Collectors.toSet()); pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber, - importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList()))); + importedEntities.stream() + .filter(i -> pageNumber == i.getEntityPosition() + .get(0).pageNumber()) + .collect(Collectors.toList()))); return importedRedactionsMap; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java index c2c4ecc9..e1efe9d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java @@ -15,8 +15,12 @@ public class ComponentComparator implements Comparator { @Override public int compare(Component component1, Component component2) { - var firstEntity1 = component1.getReferences().stream().min(EntityComparators.first()); - var firstEntity2 = component2.getReferences().stream().min(EntityComparators.first()); + var firstEntity1 = component1.getReferences() + .stream() + .min(EntityComparators.first()); + var firstEntity2 = component2.getReferences() + .stream() + .min(EntityComparators.first()); if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) { return 0; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index f40de35e..886c7240 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -40,7 +40,8 @@ public class ComponentCreationService { private static List findEntitiesFromLongestSection(Collection entities) { - var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); + var entitiesBySection = entities.stream() + .collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); Optional longestSection = entitiesBySection.entrySet() .stream() .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed()) @@ -79,14 +80,20 @@ public class ComponentCreationService { public void firstOrElse(String ruleIdentifier, String name, Collection entities, String fallback) { String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback); - String value = entities.stream().min(EntityComparators.first()).map(Entity::getValue).orElse(fallback); + String value = entities.stream() + .min(EntityComparators.first()) + .map(Entity::getValue) + .orElse(fallback); create(ruleIdentifier, name, value, valueDescription, entities); } private static String joinTypes(Collection entities) { - return entities.stream().map(Entity::getType).distinct().collect(Collectors.joining(", ")); + return entities.stream() + .map(Entity::getType) + .distinct() + .collect(Collectors.joining(", ")); } @@ -104,12 +111,12 @@ public class ComponentCreationService { referencedEntities.addAll(references); kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) - .name(name) - .value(value) - .valueDescription(valueDescription) - .references(new LinkedList<>(references)) - .build()); + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .name(name) + .value(value) + .valueDescription(valueDescription) + .references(new LinkedList<>(references)) + .build()); } @@ -142,8 +149,11 @@ public class ComponentCreationService { private static List findEntitiesFromFirstSection(Collection entities) { - var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); - Optional firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first()); + var entitiesBySection = entities.stream() + .collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent())); + Optional firstSection = entitiesBySection.keySet() + .stream() + .min(SemanticNodeComparators.first()); if (firstSection.isEmpty()) { return Collections.emptyList(); } @@ -188,7 +198,10 @@ public class ComponentCreationService { public void joining(String ruleIdentifier, String name, Collection entities, String delimiter) { String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter); - String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(delimiter)); + String value = entities.stream() + .sorted(EntityComparators.first()) + .map(Entity::getValue) + .collect(Collectors.joining(delimiter)); create(ruleIdentifier, name, value, valueDescription, entities); } @@ -231,14 +244,20 @@ public class ComponentCreationService { public void joiningUnique(String ruleIdentifier, String name, Collection entities, String delimiter) { String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter); - String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter)); + String value = entities.stream() + .sorted(EntityComparators.first()) + .map(Entity::getValue) + .distinct() + .collect(Collectors.joining(delimiter)); create(ruleIdentifier, name, value, valueDescription, entities); } private static int getTotalLengthOfEntities(Map.Entry> entry) { - return entry.getValue().stream().mapToInt(Entity::getLength).sum(); + return entry.getValue() + .stream() + .mapToInt(Entity::getLength).sum(); } @@ -293,7 +312,10 @@ public class ComponentCreationService { */ public void uniqueValueCount(String ruleIdentifier, String name, Collection entities) { - long count = entities.stream().map(Entity::getValue).distinct().count(); + long count = entities.stream() + .map(Entity::getValue) + .distinct() + .count(); create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities); } @@ -307,18 +329,20 @@ public class ComponentCreationService { */ public void rowValueCount(String ruleIdentifier, String name, Collection entities) { - entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> { + entities.stream() + .collect(Collectors.groupingBy(this::getFirstTable)) + .forEach((optionalTable, groupedEntities) -> { - if (optionalTable.isEmpty()) { - return; - } + if (optionalTable.isEmpty()) { + return; + } - long count = groupedEntities.stream() - .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1))) - .size(); + long count = groupedEntities.stream() + .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow) + .orElse(-1))).size(); - create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities); - }); + create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities); + }); } @@ -334,18 +358,20 @@ public class ComponentCreationService { if (entities.isEmpty()) { return; } - entities.stream().sorted(EntityComparators.first()).forEach(entity -> { - BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH); - iterator.setText(entity.getValue()); - int start = iterator.first(); - for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) { - create(ruleIdentifier, - name, - entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), - String.format("Values of type '%s' as sentences", entity.getType()), - entity); - } - }); + entities.stream() + .sorted(EntityComparators.first()) + .forEach(entity -> { + BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH); + iterator.setText(entity.getValue()); + int start = iterator.first(); + for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) { + create(ruleIdentifier, + name, + entity.getValue().substring(start, end).replaceAll("\\n", "").trim(), + String.format("Values of type '%s' as sentences", entity.getType()), + entity); + } + }); } @@ -366,12 +392,12 @@ public class ComponentCreationService { List referenceList = new LinkedList<>(); referenceList.add(reference); kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) - .name(name) - .value(value) - .valueDescription(valueDescription) - .references(referenceList) - .build()); + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .name(name) + .value(value) + .valueDescription(valueDescription) + .references(referenceList) + .build()); } @@ -428,8 +454,10 @@ public class ComponentCreationService { } String formattedDateStrings = Stream.concat(// - dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), // - unparsedDates.stream())// + dates.stream() + .sorted() + .map(date -> DateConverter.convertDate(date, resultFormat)), // + unparsedDates.stream())// .collect(Collectors.joining(", ")); create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities); @@ -445,26 +473,34 @@ public class ComponentCreationService { */ public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection entities) { - String types = entities.stream().map(Entity::getType).sorted(Comparator.reverseOrder()).distinct().collect(Collectors.joining(", ")); + String types = entities.stream() + .map(Entity::getType) + .sorted(Comparator.reverseOrder()) + .distinct() + .collect(Collectors.joining(", ")); String valueDescription = String.format("Combine values of %s that are in same table row", types); - entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> { - if (optionalTable.isEmpty()) { - groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity)); - } + entities.stream() + .collect(Collectors.groupingBy(this::getFirstTable)) + .forEach((optionalTable, groupedEntities) -> { + if (optionalTable.isEmpty()) { + groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity)); + } - groupedEntities.stream() - .filter(entity -> entity.getContainingNode() instanceof TableCell) - .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())) - .entrySet() - .stream() - .sorted(Comparator.comparingInt(Map.Entry::getKey)) - .map(Map.Entry::getValue) - .forEach(entitiesInSameRow -> create(ruleIdentifier, - name, - entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")), - valueDescription, - entitiesInSameRow)); - }); + groupedEntities.stream() + .filter(entity -> entity.getContainingNode() instanceof TableCell) + .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())).entrySet() + .stream() + .sorted(Comparator.comparingInt(Map.Entry::getKey)) + .map(Map.Entry::getValue) + .forEach(entitiesInSameRow -> create(ruleIdentifier, + name, + entitiesInSameRow.stream() + .sorted(EntityComparators.first()) + .map(Entity::getValue) + .collect(Collectors.joining(", ")), + valueDescription, + entitiesInSameRow)); + }); } @@ -521,12 +557,12 @@ public class ComponentCreationService { public void create(String ruleIdentifier, String name, String value) { kieSession.insert(Component.builder() - .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) - .name(name) - .value(value) - .valueDescription("") - .references(Collections.emptyList()) - .build()); + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .name(name) + .value(value) + .valueDescription("") + .references(Collections.emptyList()) + .build()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java index c8164789..1a9a3531 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java @@ -40,7 +40,9 @@ public class DocumentGraphMapper { DocumentTree documentTree = new DocumentTree(document); Context context = new Context(documentData, documentTree); - context.pageData.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList()); + context.pageData.addAll(Arrays.stream(documentData.getDocumentPages()) + .map(DocumentGraphMapper::buildPage) + .toList()); context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context)); @@ -58,7 +60,9 @@ public class DocumentGraphMapper { List newEntries = new LinkedList<>(); for (DocumentStructure.EntryData entryData : entries) { - List pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList(); + List pages = Arrays.stream(entryData.getPageNumbers()) + .map(pageNumber -> getPage(pageNumber, context)) + .toList(); SemanticNode node = switch (entryData.getType()) { case SECTION -> buildSection(context); @@ -76,8 +80,10 @@ public class DocumentGraphMapper { TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node); node.setLeafTextBlock(textBlock); } - List treeId = Arrays.stream(entryData.getTreeId()).boxed().toList(); - entryData.getEngines().forEach(engine -> node.addEngine(engine)); + List treeId = Arrays.stream(entryData.getTreeId()).boxed() + .toList(); + entryData.getEngines() + .forEach(engine -> node.addEngine(engine)); node.setTreeId(treeId); switch (entryData.getType()) { @@ -150,16 +156,18 @@ public class DocumentGraphMapper { private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) { - return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector()); + return Arrays.stream(atomicTextBlockIds) + .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)) + .collect(new TextBlockCollector()); } private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) { return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)), - context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)), - parent, - getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context)); + context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)), + parent, + getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context)); } @@ -190,8 +198,10 @@ public class DocumentGraphMapper { this.documentTree = documentTree; this.pageData = new LinkedList<>(); - this.documentTextData = Arrays.stream(documentData.getDocumentTextData()).toList(); - this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData()).toList(); + this.documentTextData = Arrays.stream(documentData.getDocumentTextData()) + .toList(); + this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData()) + .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java index 157fbfd6..f34ba236 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java @@ -11,6 +11,7 @@ public abstract class EntityComparators implements Comparator { return new FirstEntity(); } + public static class LongestEntity implements Comparator { @Override @@ -27,6 +28,7 @@ public abstract class EntityComparators implements Comparator { return new LongestEntity(); } + public static class FirstEntity implements Comparator { @Override diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index 4a9d31d4..f8b803d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -276,7 +276,8 @@ public class EntityCreationService { "this is some text. a here is more text" and "here is more text". We only want to keep the latter. */ return entityTextRanges.stream() - .filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary))) + .filter(boundary -> entityTextRanges.stream() + .noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary))) .toList(); } @@ -351,10 +352,10 @@ public class EntityCreationService { return tableNode.streamTableCells() .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()), - tableCell, - type, - entityType, - tableNode)); + tableCell, + type, + entityType, + tableNode)); } @@ -362,10 +363,10 @@ public class EntityCreationService { return tableNode.streamTableCells() .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, tableCell.getTextBlock()), - tableCell, - type, - entityType, - tableNode)); + tableCell, + type, + entityType, + tableNode)); } @@ -500,7 +501,10 @@ public class EntityCreationService { public Stream bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) { - return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get); + return node.streamAllSubNodesOfType(NodeType.PARAGRAPH) + .map(semanticNode -> bySemanticNode(semanticNode, type, entityType)) + .filter(Optional::isPresent) + .map(Optional::get); } @@ -592,7 +596,11 @@ public class EntityCreationService { TextRange trimmedTextRange = textRange.trim(node.getTextBlock()); TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node); if (node.getEntities().contains(entity)) { - Optional optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny(); + Optional optionalTextEntity = node.getEntities() + .stream() + .filter(e -> e.equals(entity) && e.type().equals(type)) + .peek(e -> e.addEngines(engines)) + .findAny(); if (optionalTextEntity.isEmpty()) { return optionalTextEntity; // Entity has been recategorized and should not be created at all. } @@ -647,17 +655,27 @@ public class EntityCreationService { return entitiesToMerge.get(0); } - TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node); - mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); - entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule)); + TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream() + .map(TextEntity::getTextRange) + .toList()), type, entityType, node); + mergedEntity.addEngines(entitiesToMerge.stream() + .flatMap(entityNode -> entityNode.getEngines() + .stream()) + .collect(Collectors.toSet())); + entitiesToMerge.stream() + .map(TextEntity::getMatchedRuleList) + .flatMap(Collection::stream) + .forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule)); entitiesToMerge.stream() .map(TextEntity::getManualOverwrite) .map(ManualChangeOverwrite::getManualChangeLog) .flatMap(Collection::stream) .forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange)); - mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(TextEntity::isDictionaryEntry)); - mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(TextEntity::isDossierDictionaryEntry)); + mergedEntity.setDictionaryEntry(entitiesToMerge.stream() + .anyMatch(TextEntity::isDictionaryEntry)); + mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream() + .anyMatch(TextEntity::isDossierDictionaryEntry)); addEntityToGraph(mergedEntity, node); insertToKieSession(mergedEntity); @@ -667,7 +685,8 @@ public class EntityCreationService { public Stream copyEntities(List entities, String type, EntityType entityType, SemanticNode node) { - return entities.stream().map(entity -> copyEntity(entity, type, entityType, node)); + return entities.stream() + .map(entity -> copyEntity(entity, type, entityType, node)); } @@ -744,7 +763,8 @@ public class EntityCreationService { try { if (node.getEntities().contains(entity)) { // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges - node.getEntities().stream()// + node.getEntities() + .stream()// .filter(e -> e.equals(entity))// .filter(e -> !e.getTextRange().equals(entity.getTextRange()))// .findAny()// @@ -770,8 +790,10 @@ public class EntityCreationService { SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes() .stream() .sorted(Comparator.comparingInt(n -> -n.getTreeId().size())) - .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && // - intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange())) + .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges() + .stream() + .allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && // + intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange())) .findFirst() .orElse(node.getDocumentTree().getRoot().getNode()); @@ -784,7 +806,8 @@ public class EntityCreationService { return; } additionalIntersectingNode.getEntities().add(entityToDuplicate); - additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate)); + additionalIntersectingNode.getPages(newTextRange) + .forEach(page -> page.getEntities().add(entityToDuplicate)); entityToDuplicate.addIntersectingNode(additionalIntersectingNode); }); } @@ -806,5 +829,4 @@ public class EntityCreationService { addEntityToNodeEntitySets(entity); } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java index 25f1b0d1..59c3fd4c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java @@ -11,7 +11,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl public class EntityCreationUtility { - public static void checkIfBothStartAndEndAreEmpty(String start, String end) { checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); @@ -57,7 +56,8 @@ public class EntityCreationUtility { public static void addEntityToNodeEntitySets(TextEntity entity) { - entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); + entity.getIntersectingNodes() + .forEach(node -> node.getEntities().add(entity)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java index 586ab810..f39bb722 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java @@ -59,7 +59,9 @@ public class EntityEnrichmentService { private static List splitToWordsAndRemoveEmptyWords(String textAfter) { - return Arrays.stream(textAfter.split(" ")).filter(word -> !Objects.equals("", word)).toList(); + return Arrays.stream(textAfter.split(" ")) + .filter(word -> !Objects.equals("", word)) + .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java index 76b9cfb2..b070ed71 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java @@ -47,7 +47,9 @@ public class EntityFindingUtility { } - public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map> entitiesWithSameValue, double matchThreshold) { + public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, + Map> entitiesWithSameValue, + double matchThreshold) { if (precursorEntity.getValue() == null) { return Optional.empty(); @@ -73,11 +75,15 @@ public class EntityFindingUtility { ClosestEntity closestEntity = optionalClosestEntity.get(); if (closestEntity.getDistance() > matchThreshold) { log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", - precursorEntity.getValue(), - precursorEntity.getEntityPosition().get(0).pageNumber(), - precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), - closestEntity.getDistance(), - matchThreshold); + precursorEntity.getValue(), + precursorEntity.getEntityPosition() + .get(0).pageNumber(), + precursorEntity.getEntityPosition() + .stream() + .map(RectangleWithPage::rectangle2D) + .toList(), + closestEntity.getDistance(), + matchThreshold); return Optional.empty(); } @@ -93,8 +99,14 @@ public class EntityFindingUtility { private static boolean pagesMatch(TextEntity entity, List originalPositions) { - Set entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet()); - Set originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet()); + Set entityPageNumbers = entity.getPositionsOnPagePerPage() + .stream() + .map(PositionOnPage::getPage) + .map(Page::getNumber) + .collect(Collectors.toSet()); + Set originalPageNumbers = originalPositions.stream() + .map(RectangleWithPage::pageNumber) + .collect(Collectors.toSet()); return entityPageNumbers.containsAll(originalPageNumbers); } @@ -105,15 +117,16 @@ public class EntityFindingUtility { return Double.MAX_VALUE; } return originalPositions.stream() - .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())) - .average() + .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average() .orElse(Double.MAX_VALUE); } private static long countRectangles(TextEntity entity) { - return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum(); + return entity.getPositionsOnPagePerPage() + .stream() + .mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum(); } @@ -144,22 +157,33 @@ public class EntityFindingUtility { double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY()); return Math.abs(minX1 - minX2) // - + Math.abs(minY1 - minY2) // - + Math.abs(maxX1 - maxX2) // - + Math.abs(maxY1 - maxY2); + + Math.abs(minY1 - minY2) // + + Math.abs(maxX1 - maxX2) // + + Math.abs(maxY1 - maxY2); } public Map> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List manualEntities) { - Set pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet()); - Set entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet()); + Set pageNumbers = manualEntities.stream() + .flatMap(entry -> entry.getEntityPosition() + .stream() + .map(RectangleWithPage::pageNumber)) + .collect(Collectors.toSet()); + Set entryValues = manualEntities.stream() + .map(PrecursorEntity::getValue) + .filter(Objects::nonNull) + .map(String::toLowerCase) + .collect(Collectors.toSet()); - if (!pageNumbers.stream().allMatch(node::onPage)) { + if (!pageNumbers.stream() + .allMatch(node::onPage)) { throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s", - node, - pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(), - node.getPages())); + node, + pageNumbers.stream() + .filter(pageNumber -> !node.onPage(pageNumber)) + .toList(), + node.getPages())); } SearchImplementation searchImplementation = new SearchImplementation(entryValues, true); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java index 8c5015d1..5fc647df 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java @@ -52,9 +52,13 @@ public class ImportedRedactionEntryService { private List allManualChangesExceptAdd(ManualRedactions manualRedactions) { return Stream.of(manualRedactions.getForceRedactions(), - manualRedactions.getResizeRedactions(), - manualRedactions.getRecategorizations(), - manualRedactions.getIdsToRemove(), - manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); + manualRedactions.getResizeRedactions(), + manualRedactions.getRecategorizations(), + manualRedactions.getIdsToRemove(), + manualRedactions.getLegalBasisChanges()) + .flatMap(Collection::stream) + .map(baseAnnotation -> (BaseAnnotation) baseAnnotation) + .toList(); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java index 6b5ac081..5c76e6a1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java @@ -14,12 +14,14 @@ public class IntersectingNodeVisitor implements NodeVisitor { private Set intersectingNodes; private final TextRange textRange; + public IntersectingNodeVisitor(TextRange textRange) { this.textRange = textRange; this.intersectingNodes = new HashSet<>(); } + @Override public void visit(SemanticNode node) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java index d9f82c86..3900ef21 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java @@ -31,7 +31,8 @@ public class ManualRedactionEntryService { List notFoundManualRedactionEntries = Collections.emptyList(); if (analyzeRequest.getManualRedactions() != null) { notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(), - document, dossierTemplateId); + document, + dossierTemplateId); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } @@ -51,10 +52,13 @@ public class ManualRedactionEntryService { private List allManualChangesExceptAdd(ManualRedactions manualRedactions) { return Stream.of(manualRedactions.getForceRedactions(), - manualRedactions.getResizeRedactions(), - manualRedactions.getRecategorizations(), - manualRedactions.getIdsToRemove(), - manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); + manualRedactions.getResizeRedactions(), + manualRedactions.getRecategorizations(), + manualRedactions.getIdsToRemove(), + manualRedactions.getLegalBasisChanges()) + .flatMap(Collection::stream) + .map(baseAnnotation -> (BaseAnnotation) baseAnnotation) + .toList(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java index 5d4096fe..5fa03602 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java @@ -44,9 +44,11 @@ public class NerEntitiesAdapter { public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document) { return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(document), - nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(), - new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()), - nerEntityModel.getType())).toList()); + nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(), + new TextRange(nerEntityModel.getStartOffset(), + nerEntityModel.getEndOffset()), + nerEntityModel.getType())) + .toList()); } @@ -83,7 +85,9 @@ public class NerEntitiesAdapter { List> entityClusters = new LinkedList<>(); - List startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> essentialTypes.contains(e.type())).toList(); + List startEntitiesOfEssentialType = sortedEntities.stream() + .filter(e -> essentialTypes.contains(e.type())) + .toList(); for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) { List currentCluster = new LinkedList<>(); entityClusters.add(currentCluster); @@ -105,7 +109,10 @@ public class NerEntitiesAdapter { } } - return entityClusters.stream().filter(cluster -> cluster.size() >= minPartsToCombine).map(NerEntitiesAdapter::toContainingBoundary).distinct(); + return entityClusters.stream() + .filter(cluster -> cluster.size() >= minPartsToCombine) + .map(NerEntitiesAdapter::toContainingBoundary) + .distinct(); } @@ -124,17 +131,18 @@ public class NerEntitiesAdapter { public Stream combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) { return combineNerEntities(entityRecognitionEntities, - CBI_ADDRESS_ESSENTIAL_TYPES, - CBI_ADDRESS_TYPES_TO_COMBINE, - MAX_DISTANCE_BETWEEN_PARTS, - MIN_PARTS_TO_COMBINE, - ALLOW_DUPLICATES); + CBI_ADDRESS_ESSENTIAL_TYPES, + CBI_ADDRESS_TYPES_TO_COMBINE, + MAX_DISTANCE_BETWEEN_PARTS, + MIN_PARTS_TO_COMBINE, + ALLOW_DUPLICATES); } private static boolean isDuplicate(List currentCluster, NerEntities.NerEntity entity, boolean allowDuplicates) { - return allowDuplicates || currentCluster.stream().anyMatch(e -> e.type().equals(entity.type())); + return allowDuplicates || currentCluster.stream() + .anyMatch(e -> e.type().equals(entity.type())); } @@ -146,24 +154,34 @@ public class NerEntitiesAdapter { private static TextRange toContainingBoundary(List nerEntities) { - return TextRange.merge(nerEntities.stream().map(NerEntities.NerEntity::textRange).toList()); + return TextRange.merge(nerEntities.stream() + .map(NerEntities.NerEntity::textRange) + .toList()); } private static Stream addOffsetsAndFlatten(List stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) { - nerEntitiesModel.getData().forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> { - int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber); - entityRecognitionEntity.setStartOffset(newStartOffset); - entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); - })); - return nerEntitiesModel.getData().values().stream().flatMap(Collection::stream); + nerEntitiesModel.getData() + .forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> { + int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber); + entityRecognitionEntity.setStartOffset(newStartOffset); + entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); + })); + return nerEntitiesModel.getData().values() + .stream() + .flatMap(Collection::stream); } private static List getStringStartOffsetsForMainSections(Document document) { - return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getTextRange).map(TextRange::start).toList(); + return document.getMainSections() + .stream() + .map(Section::getTextBlock) + .map(TextBlock::getTextRange) + .map(TextRange::start) + .toList(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java index a027c47a..53436c91 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java @@ -5,4 +5,5 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo public interface NodeVisitor { void visit(SemanticNode node); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java index 220a35de..dec16250 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java @@ -43,7 +43,9 @@ public class PropertiesMapper { private Rectangle2D parseRectangle2D(String bBox) { - List floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER)).map(Float::parseFloat).toList(); + List floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER)) + .map(Float::parseFloat) + .toList(); return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java index 165d8c15..65ddd841 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java @@ -12,6 +12,7 @@ public abstract class SemanticNodeComparators implements Comparator entityLogEntry.getState().equals(EntryState.APPLIED)) .map(entry -> Entity.fromEntityLogEntry(entry, document)) .forEach(kieSession::insert); - fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); + fileAttributes.stream() + .filter(f -> f.getValue() != null) + .forEach(kieSession::insert); CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { kieSession.fireAllRules(); @@ -58,7 +60,8 @@ public class ComponentDroolsExecutionService { }); try { - completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get(); + completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS) + .get(); } catch (ExecutionException e) { kieSession.dispose(); if (e.getCause() instanceof TimeoutException) { @@ -71,7 +74,9 @@ public class ComponentDroolsExecutionService { } List resultingFileAttributes = getFileAttributes(kieSession); - List components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList(); + List components = getComponents(kieSession).stream() + .sorted(ComponentComparator.first()) + .toList(); kieSession.dispose(); return components; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java index 4aa89dda..1e1fe0ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java @@ -60,6 +60,7 @@ public class DroolsSyntaxValidationService { return droolsCompilerSyntaxValidation; } + private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) { if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) { @@ -70,13 +71,13 @@ public class DroolsSyntaxValidationService { String sb = "Following imports are deprecated: \n" + matches.stream() .map(m -> imports.substring(m.startIndex(), m.endIndex())) .collect(Collectors.joining("\n")); - return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb) - .build(); + return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb).build(); } } return null; } + private List getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) { List warningMessages = new ArrayList<>(); @@ -96,8 +97,7 @@ public class DroolsSyntaxValidationService { .distinct() .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm))) .collect(Collectors.joining("\n")); - warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage) - .build()); + warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage).build()); } } } @@ -160,7 +160,10 @@ public class DroolsSyntaxValidationService { String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS"; if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) { customSyntaxValidation.getDroolsSyntaxErrorMessages() - .add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup)) + .add(DroolsSyntaxErrorMessage.builder() + .line(0) + .column(0) + .message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup)) .build()); } } @@ -214,15 +217,13 @@ public class DroolsSyntaxValidationService { List droolsSyntaxErrorMessages = errorMessages.stream() .map(this::buildDroolsSyntaxErrorMessage) .collect(Collectors.toList()); - return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages) - .build(); + return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build(); } private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) { - return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()) - .build(); + return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java index 1e9e3f92..8878f5b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java @@ -30,8 +30,7 @@ public class KieContainerCreationService { private final RulesClient rulesClient; - @Observed(name = "KieContainerCreationService", - contextualName = "get-kie-container") + @Observed(name = "KieContainerCreationService", contextualName = "get-kie-container") public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) { try { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java index 5eac41a3..0c4ed5f8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java @@ -52,7 +52,12 @@ public class RuleFileParser { } } - String imports = ruleString.substring(0, packageDescr.getImports().stream().mapToInt(ImportDescr::getEndCharacter).max().orElseThrow() + 1); + String imports = ruleString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -61,11 +66,20 @@ public class RuleFileParser { List ruleClasses = buildRuleClasses(allRules); return new RuleFileBluePrint(imports.trim(), - packageDescr.getImports().stream().findFirst().map(ImportDescr::getLine).orElse(0), - globals.trim(), - packageDescr.getGlobals().stream().findFirst().map(GlobalDescr::getLine).orElse(0), allQueries, - ruleClasses, - customDroolsSyntaxValidation); + packageDescr.getImports() + .stream() + .findFirst() + .map(ImportDescr::getLine) + .orElse(0), + globals.trim(), + packageDescr.getGlobals() + .stream() + .findFirst() + .map(GlobalDescr::getLine) + .orElse(0), + allQueries, + ruleClasses, + customDroolsSyntaxValidation); } @@ -91,11 +105,12 @@ public class RuleFileParser { Matcher matcher = ruleIdentifierInCodeFinder.matcher(code); while (matcher.find()) { String identifierInCode = code.substring(matcher.start(1), matcher.end(1)); - long line = code.substring(0, matcher.start(1)).lines().count() + lineOffset - 1; + long line = code.substring(0, matcher.start(1)).lines() + .count() + lineOffset - 1; if (!identifier.equals(identifierInCode)) { customDroolsSyntaxValidation.addErrorMessage((int) line, - 0, - String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier)); + 0, + String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier)); } } } @@ -104,23 +119,35 @@ public class RuleFileParser { private void addDuplicateRuleIdentifierErrorMessage(RuleDescr rule, BasicRule basicRule, DroolsSyntaxValidation customDroolsSyntaxValidation) { customDroolsSyntaxValidation.addErrorMessage(rule.getLine(), - rule.getColumn(), - String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier())); + rule.getColumn(), + String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier())); } private List buildRuleClasses(List allRules) { - List ruleTypeOrder = allRules.stream().map(BasicRule::getIdentifier).map(RuleIdentifier::type).distinct().toList(); - Map> rulesPerType = allRules.stream().collect(groupingBy(rule -> rule.getIdentifier().type())); - return ruleTypeOrder.stream().map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))).collect(Collectors.toList()); + List ruleTypeOrder = allRules.stream() + .map(BasicRule::getIdentifier) + .map(RuleIdentifier::type) + .distinct() + .toList(); + Map> rulesPerType = allRules.stream() + .collect(groupingBy(rule -> rule.getIdentifier().type())); + return ruleTypeOrder.stream() + .map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))) + .collect(Collectors.toList()); } private List groupingByGroup(List rules) { - Map> rulesPerUnit = rules.stream().collect(groupingBy(rule -> rule.getIdentifier().unit())); - return rulesPerUnit.keySet().stream().sorted().map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))).collect(Collectors.toList()); + Map> rulesPerUnit = rules.stream() + .collect(groupingBy(rule -> rule.getIdentifier().unit())); + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/ObservedStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/ObservedStorageService.java index 81641192..becdb04b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/ObservedStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/ObservedStorageService.java @@ -16,6 +16,7 @@ public class ObservedStorageService { @Observed(name = "RedactionStorageService", contextualName = "get-document-data") public DocumentData getDocumentData(String dossierId, String fileId) { + return redactionStorageService.getDocumentData(dossierId, fileId); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 0aaeaac9..08044c93 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -80,9 +80,12 @@ public class RedactionStorageService { try { ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), - ImportedRedactionsPerPage.class); - return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values().stream().flatMap(List::stream).collect(Collectors.toList())); + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), + ImportedRedactionsPerPage.class); + return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList())); } catch (StorageObjectDoesNotExist e) { log.debug("Imported redactions not available."); return new ImportedRedactions(); @@ -90,14 +93,13 @@ public class RedactionStorageService { } - @Timed("redactmanager_getImportedRedactions") public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) { try { return storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), - ImportedRedactionsPerPage.class); + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), + ImportedRedactionsPerPage.class); } catch (StorageObjectDoesNotExist e) { log.debug("Imported redactions not available."); return null; @@ -111,12 +113,12 @@ public class RedactionStorageService { try { RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), - RedactionLog.class); + StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), + RedactionLog.class); redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) - .collect(Collectors.toList())); + .stream() + .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) + .collect(Collectors.toList())); return redactionLog; } catch (StorageObjectDoesNotExist e) { log.debug("RedactionLog not available."); @@ -132,9 +134,9 @@ public class RedactionStorageService { try { EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); entityLog.setEntityLogEntry(entityLog.getEntityLogEntry() - .stream() - .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) - .collect(Collectors.toList())); + .stream() + .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty())) + .collect(Collectors.toList())); return entityLog; } catch (StorageObjectDoesNotExist e) { log.debug("EntityLog not available."); @@ -156,17 +158,17 @@ public class RedactionStorageService { try { return DocumentData.builder() .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), - DocumentStructure.class)) + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), + DocumentStructure.class)) .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), - DocumentTextData[].class)) + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), + DocumentTextData[].class)) .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), - DocumentPositionData[].class)) + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), + DocumentPositionData[].class)) .documentPages(storageService.readJSONObject(TenantContext.getTenantId(), - StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), - DocumentPage[].class)) + StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), + DocumentPage[].class)) .build(); } catch (StorageObjectDoesNotExist e) { log.debug("DocumentData not available."); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java index a4d5e00e..e320ae7a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java @@ -11,6 +11,7 @@ public class RuleManagementResources { private static final String folderPrefix = "drools"; + @SneakyThrows public static InputStream getBaseRuleFileInputStream() { @@ -26,6 +27,7 @@ public class RuleManagementResources { } } + @SneakyThrows public static InputStream getBaseComponentRuleFileInputStream() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java index ccf8d832..9e568ce8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java @@ -18,11 +18,11 @@ import lombok.extern.slf4j.Slf4j; public class DateConverter { static List formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH), - new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH), - new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH), - new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH), - new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH), - new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH)); + new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH), + new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH), + new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH), + new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH), + new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH)); public Optional parseDate(String dateAsString) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java index 61b0d430..3e5258aa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java @@ -21,7 +21,9 @@ public final class IdBuilder { public String buildId(Set pages, List rectanglesPerLine, String type, String entityType) { - return buildId(pages.stream().map(Page::getNumber).collect(Collectors.toList()), rectanglesPerLine, type, entityType); + return buildId(pages.stream() + .map(Page::getNumber) + .collect(Collectors.toList()), rectanglesPerLine, type, entityType); } @@ -29,7 +31,9 @@ public final class IdBuilder { StringBuilder sb = new StringBuilder(); sb.append(type).append(entityType); - List sortedPageNumbers = pageNumbers.stream().sorted(Comparator.comparingInt(Integer::intValue)).toList(); + List sortedPageNumbers = pageNumbers.stream() + .sorted(Comparator.comparingInt(Integer::intValue)) + .toList(); sortedPageNumbers.forEach(sb::append); rectanglesPerLine.forEach(rectangle2D -> sb.append(Math.round(rectangle2D.getX())) .append(Math.round(rectangle2D.getY())) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java index 6e30eb46..d4746f21 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java @@ -22,19 +22,25 @@ public class RectangleTransformations { public static Rectangle2D atomicTextBlockBBox(List atomicTextBlocks) { - return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector()); + return atomicTextBlocks.stream() + .flatMap(atomicTextBlock -> atomicTextBlock.getPositions() + .stream()) + .collect(new Rectangle2DBBoxCollector()); } public static Rectangle2D rectangleBBox(List positions) { - return positions.stream().map(Position::toRectangle2D).collect(new Rectangle2DBBoxCollector()); + return positions.stream() + .map(Position::toRectangle2D) + .collect(new Rectangle2DBBoxCollector()); } public static Rectangle2D rectangle2DBBox(List rectangle2DList) { - return rectangle2DList.stream().collect(new Rectangle2DBBoxCollector()); + return rectangle2DList.stream() + .collect(new Rectangle2DBBoxCollector()); } @@ -49,7 +55,9 @@ public class RectangleTransformations { if (rectangle2DList.isEmpty()) { return Collections.emptyList(); } - double splitThreshold = rectangle2DList.stream().mapToDouble(RectangularShape::getWidth).average().orElse(5) * 5.0; + double splitThreshold = rectangle2DList.stream() + .mapToDouble(RectangularShape::getWidth).average() + .orElse(5) * 5.0; List> rectangleListsWithGaps = new LinkedList<>(); List rectangleListWithoutGaps = new LinkedList<>(); @@ -66,7 +74,9 @@ public class RectangleTransformations { previousRectangle = currentRectangle; } } - return rectangleListsWithGaps.stream().map(RectangleTransformations::rectangle2DBBox).toList(); + return rectangleListsWithGaps.stream() + .map(RectangleTransformations::rectangle2DBBox) + .toList(); } @@ -96,9 +106,9 @@ public class RectangleTransformations { public BinaryOperator combiner() { return (b1, b2) -> new BBox(Math.min(b1.lowerLeftX, b2.lowerLeftX), - Math.min(b1.lowerLeftY, b2.lowerLeftY), - Math.max(b1.upperRightX, b2.upperRightX), - Math.max(b1.upperRightY, b2.upperRightY)); + Math.min(b1.lowerLeftY, b2.lowerLeftY), + Math.max(b1.upperRightX, b2.upperRightX), + Math.max(b1.upperRightY, b2.upperRightY)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java index e5bcba11..d8dc287b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java @@ -89,7 +89,10 @@ public class RedactionSearchUtility { public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) { - List lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed().map(textBlock::getLineTextRange).filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary)).toList(); + List lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed() + .map(textBlock::getLineTextRange) + .filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary)) + .toList(); if (lineBoundaries.isEmpty()) { return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start()); } @@ -132,6 +135,7 @@ public class RedactionSearchUtility { return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern); } + public static List findTextRangesByRegexIgnoreCase(String regexPattern, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, true); @@ -185,19 +189,21 @@ public class RedactionSearchUtility { return getTextRangesByPattern(textBlock, 0, pattern); } + public static List findTextRangesByList(List searchList, TextBlock textBlock) { List boundaries = new LinkedList<>(); - for (var searchString: searchList) { + for (var searchString : searchList) { boundaries.addAll(findTextRangesByString(searchString, textBlock)); } return boundaries; } + public static List findTextRangesByListIgnoreCase(List searchList, TextBlock textBlock) { List boundaries = new LinkedList<>(); - for (var searchString: searchList) { + for (var searchString : searchList) { boundaries.addAll(findTextRangesByStringIgnoreCase(searchString, textBlock)); } return boundaries; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java index 2cf7258b..54160add 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java @@ -20,7 +20,8 @@ public final class ResourceLoader { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); } try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - return br.lines().collect(Collectors.toSet()); + return br.lines() + .collect(Collectors.toSet()); } catch (IOException e) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java index 5145876b..391a98aa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java @@ -54,16 +54,16 @@ public final class SeparatorUtils { private static boolean validateEnd(TextBlock textBlock, TextRange textRange) { return textRange.end() == textBlock.getTextRange().end() ||// - SeparatorUtils.isSeparator(textBlock.charAt(textRange.end())) ||// - SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.end() - 1)); + SeparatorUtils.isSeparator(textBlock.charAt(textRange.end())) ||// + SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.end() - 1)); } private static boolean validateStart(TextBlock textBlock, TextRange textRange) { return textRange.start() == textBlock.getTextRange().start() ||// - SeparatorUtils.isSeparator(textBlock.charAt(textRange.start() - 1)) ||// - SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.start())); + SeparatorUtils.isSeparator(textBlock.charAt(textRange.start() - 1)) ||// + SeparatorUtils.isJapaneseSeparator(textBlock.charAt(textRange.start())); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index d462c2eb..f37de123 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -172,7 +172,7 @@ public abstract class AbstractRedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, - false)); + false)); when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_PII, false)); when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PURITY_INDICATOR, false)); @@ -183,9 +183,9 @@ public abstract class AbstractRedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR, - true)); + true)); when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR, - true)); + true)); } @@ -193,45 +193,102 @@ public abstract class AbstractRedactionIntegrationTest { protected void loadDictionaryForTest() { dictionary.computeIfAbsent(DICTIONARY_AUTHOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_author.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(DICTIONARY_SPONSOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(VERTEBRATE_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/vertebrate.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(DICTIONARY_ADDRESS, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_address.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(HINT_ONLY_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/hint_only.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(MUST_REDACT_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/must_redact.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(PUBLISHED_INFORMATION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/published_information.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(TEST_METHOD_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/test_method.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/PII.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(PURITY_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/purity.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(IMAGE_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(OCR_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(LOGO_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(SIGNATURE_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(FORMULA_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>()); falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); } @@ -240,7 +297,10 @@ public abstract class AbstractRedactionIntegrationTest { dictionary.clear(); dictionary.computeIfAbsent(ROTATE_SIMPLE_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); } @@ -373,8 +433,8 @@ public abstract class AbstractRedactionIntegrationTest { ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), - responseJson.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), + responseJson.getInputStream()); } @@ -424,7 +484,9 @@ public abstract class AbstractRedactionIntegrationTest { if (entries == null) { return Collections.emptyList(); } - return entries.stream().map(this::toDictionaryEntry).collect(Collectors.toList()); + return entries.stream() + .map(this::toDictionaryEntry) + .collect(Collectors.toList()); } @@ -480,11 +542,11 @@ public abstract class AbstractRedactionIntegrationTest { storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), - cvServiceResponseFileStream); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), + cvServiceResponseFileStream); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), - imageServiceResponseStream); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), + imageServiceResponseStream); return request; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java index 73481060..b64778d2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java @@ -104,15 +104,15 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index d0898053..1fdcbc73 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -82,18 +82,18 @@ public class DictionaryServiceTest { when(dictionaryClient.getVersion(anyString())).thenReturn(0L); when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc")); + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc")); var type1 = new Type(); type1.setType("type1"); @@ -101,7 +101,9 @@ public class DictionaryServiceTest { type1.setVersion(1L); type1.setHexColor("#cccccc"); type1.setHasDictionary(true); - type1.setEntries(Stream.of("a", "b", "c").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList())); + type1.setEntries(Stream.of("a", "b", "c") + .map(t -> new DictionaryEntry(1, t, 1L, false, "type1")) + .collect(Collectors.toList())); var type2 = new Type(); type2.setType("type2"); @@ -109,7 +111,9 @@ public class DictionaryServiceTest { type2.setVersion(1L); type2.setHexColor("#cccccc"); type2.setHasDictionary(true); - type2.setEntries(Stream.of("d", "e", "f").map(t -> new DictionaryEntry(1, t, 1L, false, "type2")).collect(Collectors.toList())); + type2.setEntries(Stream.of("d", "e", "f") + .map(t -> new DictionaryEntry(1, t, 1L, false, "type2")) + .collect(Collectors.toList())); var type1Updated = new Type(); type1Updated.setType("type1"); @@ -117,7 +121,9 @@ public class DictionaryServiceTest { type1Updated.setVersion(2L); type1Updated.setHexColor("#cccccc"); type1Updated.setHasDictionary(true); - type1Updated.setEntries(Stream.of("z", "q", "x").map(t -> new DictionaryEntry(1, t, 2L, false, "type1")).collect(Collectors.toList())); + type1Updated.setEntries(Stream.of("z", "q", "x") + .map(t -> new DictionaryEntry(1, t, 2L, false, "type1")) + .collect(Collectors.toList())); when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1, type2)); when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(type1); @@ -174,18 +180,18 @@ public class DictionaryServiceTest { when(dictionaryClient.getVersion(anyString())).thenReturn(0L); when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dtId", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc", - "#cccccc")); + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc")); var type = "type"; var dtType = new Type(); @@ -195,7 +201,9 @@ public class DictionaryServiceTest { dtType.setHexColor("#cccccc"); dtType.setHasDictionary(true); dtType.setDossierTemplateId("dtId"); - List dossierTemplateEntries = Stream.of("aa", "bb").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList()); + List dossierTemplateEntries = Stream.of("aa", "bb") + .map(t -> new DictionaryEntry(1, t, 1L, false, "type1")) + .collect(Collectors.toList()); dossierTemplateEntries.add(new DictionaryEntry(1, "cc", 2L, false, "type1")); assertThat(dossierTemplateEntries.size()).isEqualTo(3); dtType.setEntries(dossierTemplateEntries); @@ -208,7 +216,9 @@ public class DictionaryServiceTest { dossierType.setHasDictionary(true); dossierType.setDossierTemplateId("dtId"); dossierType.setDossierId("dossierId"); - List dossierEntries = Stream.of("aa", "bb").map(t -> new DictionaryEntry(1, t, 2L, true, "dossierType")).collect(Collectors.toList()); + List dossierEntries = Stream.of("aa", "bb") + .map(t -> new DictionaryEntry(1, t, 2L, true, "dossierType")) + .collect(Collectors.toList()); dossierEntries.add(new DictionaryEntry(1, "dd", 1L, false, "dossierType")); assertThat(dossierEntries.size()).isEqualTo(3); dossierType.setEntries(dossierEntries); @@ -224,15 +234,17 @@ public class DictionaryServiceTest { dictionaryService.updateDictionary("dtId", "dossierId"); var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId"); assertThat(dict.getDictionaryModels().size()).isEqualTo(1); - var dictModel = dict.getDictionaryModels().get(0); + var dictModel = dict.getDictionaryModels() + .get(0); assertThat(dictModel.getType()).isEqualTo(type); assertThat(dictModel.getEntries().size()).isEqualTo(4); - dictModel.getEntries().forEach(entry -> { - switch (entry.getValue()) { - case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId()); - case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()); - } - }); + dictModel.getEntries() + .forEach(entry -> { + switch (entry.getValue()) { + case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId()); + case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()); + } + }); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index ae39ce49..16430778 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -8,6 +8,7 @@ import java.util.List; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -55,9 +56,11 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf"); - ClassPathResource importedRedactionClasspathResource = new ClassPathResource("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json"); - storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactionClasspathResource.getInputStream()); - + ClassPathResource importedRedactionClasspathResource = new ClassPathResource( + "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json"); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), + importedRedactionClasspathResource.getInputStream()); // AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf", // "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json"); @@ -86,7 +89,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { // FIXME TableNodeFactory: 36, why has table no rows/cols here. AnalyzeRequest request = prepareStorage("files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.pdf", - "files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.TABLES.json"); + "files/Documine/Flora/A8591B/2009773e2e05919bb9e46aeedcc8b924.ORIGIN.TABLES.json"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request); @@ -166,15 +169,15 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 3f7726bc..abb738de 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -87,15 +87,15 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -122,6 +122,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A."); } + @Test public void acceptanceTests() throws IOException { @@ -133,8 +134,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow(); + var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() + .orElseThrow(); + var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst() + .orElseThrow(); assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); @@ -146,8 +149,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); - var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow(); + var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() + .orElseThrow(); + var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst() + .orElseThrow(); assertEquals(EntryState.APPLIED, asyaLyon2.getState()); @@ -168,13 +173,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { .stream() .filter(entry -> entry.getType().equals(type)) .filter(entry -> entry.getValue().equals(value)) - .filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0))); + .filter(entry -> entry.getContainingNodeId() + .get(0).equals(sectionNumber.get(0))); } private static Stream findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) { - return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value)); + return redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getType().equals(type)) + .filter(entry -> entry.getValue().equals(value)); } @@ -201,7 +210,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); assertEquals(EntryState.IGNORED, - findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState()); + findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)) + .findFirst() + .get().getState()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index f19e52ff..cc1400b2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -78,15 +78,15 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -128,7 +128,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(1); - EntityLogEntry redactionLogEntry = entityLog.getEntityLogEntry().get(0); + EntityLogEntry redactionLogEntry = entityLog.getEntityLogEntry() + .get(0); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorAndPIIDictionary); @@ -164,7 +165,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLog.getEntityLogEntry().size()).isEqualTo(1); - EntityLogEntry redactionLogEntry = redactionLog.getEntityLogEntry().get(0); + EntityLogEntry redactionLogEntry = redactionLog.getEntityLogEntry() + .get(0); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorDictionary); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index bd03c9c0..ee802177 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -136,99 +136,99 @@ public class RulesTest { private final Map> dossierDictionary = new HashMap<>(); private final Map> falsePositive = new HashMap<>(); private static final Map typeColorMap = Map.ofEntries(entry(VERTEBRATE, "#ff85f7"), - entry(ADDRESS, "#ffe187"), - entry(AUTHOR, "#ffe187"), - entry(SPONSOR, "#85ebff"), - entry(NO_REDACTION_INDICATOR, "#be85ff"), - entry(REDACTION_INDICATOR, "#caff85"), - entry(HINT_ONLY, "#abc0c4"), - entry(MUST_REDACT, "#fab4c0"), - entry(PUBLISHED_INFORMATION, "#85ebff"), - entry(TEST_METHOD, "#91fae8"), - entry(PII, "#66ccff"), - entry(PURITY, "#ffe187"), - entry(IMAGE, "#fcc5fb"), - entry(OCR, "#fcc5fb"), - entry(LOGO, "#ffe187"), - entry(FORMULA, "#ffe187"), - entry(SIGNATURE, "#ffe187"), - entry(IMPORTED_REDACTION, "#fcfbe6")); + entry(ADDRESS, "#ffe187"), + entry(AUTHOR, "#ffe187"), + entry(SPONSOR, "#85ebff"), + entry(NO_REDACTION_INDICATOR, "#be85ff"), + entry(REDACTION_INDICATOR, "#caff85"), + entry(HINT_ONLY, "#abc0c4"), + entry(MUST_REDACT, "#fab4c0"), + entry(PUBLISHED_INFORMATION, "#85ebff"), + entry(TEST_METHOD, "#91fae8"), + entry(PII, "#66ccff"), + entry(PURITY, "#ffe187"), + entry(IMAGE, "#fcc5fb"), + entry(OCR, "#fcc5fb"), + entry(LOGO, "#ffe187"), + entry(FORMULA, "#ffe187"), + entry(SIGNATURE, "#ffe187"), + entry(IMPORTED_REDACTION, "#fcfbe6")); private static final Map hintTypeMap = Map.ofEntries(entry(VERTEBRATE, true), - entry(ADDRESS, false), - entry(AUTHOR, false), - entry(SPONSOR, false), - entry(NO_REDACTION_INDICATOR, true), - entry(REDACTION_INDICATOR, true), - entry(HINT_ONLY, true), - entry(MUST_REDACT, true), - entry(PUBLISHED_INFORMATION, true), - entry(TEST_METHOD, true), - entry(PII, false), - entry(PURITY, false), - entry(IMAGE, true), - entry(OCR, true), - entry(FORMULA, false), - entry(LOGO, false), - entry(SIGNATURE, false), - entry(DOSSIER_REDACTIONS, false), - entry(IMPORTED_REDACTION, false)); + entry(ADDRESS, false), + entry(AUTHOR, false), + entry(SPONSOR, false), + entry(NO_REDACTION_INDICATOR, true), + entry(REDACTION_INDICATOR, true), + entry(HINT_ONLY, true), + entry(MUST_REDACT, true), + entry(PUBLISHED_INFORMATION, true), + entry(TEST_METHOD, true), + entry(PII, false), + entry(PURITY, false), + entry(IMAGE, true), + entry(OCR, true), + entry(FORMULA, false), + entry(LOGO, false), + entry(SIGNATURE, false), + entry(DOSSIER_REDACTIONS, false), + entry(IMPORTED_REDACTION, false)); private static final Map caseInSensitiveMap = Map.ofEntries(entry(VERTEBRATE, true), - entry(ADDRESS, false), - entry(AUTHOR, false), - entry(SPONSOR, false), - entry(NO_REDACTION_INDICATOR, true), - entry(REDACTION_INDICATOR, true), - entry(HINT_ONLY, true), - entry(MUST_REDACT, true), - entry(PUBLISHED_INFORMATION, true), - entry(TEST_METHOD, false), - entry(PII, false), - entry(PURITY, false), - entry(IMAGE, true), - entry(OCR, true), - entry(SIGNATURE, true), - entry(LOGO, true), - entry(FORMULA, true), - entry(DOSSIER_REDACTIONS, false), - entry(IMPORTED_REDACTION, false)); + entry(ADDRESS, false), + entry(AUTHOR, false), + entry(SPONSOR, false), + entry(NO_REDACTION_INDICATOR, true), + entry(REDACTION_INDICATOR, true), + entry(HINT_ONLY, true), + entry(MUST_REDACT, true), + entry(PUBLISHED_INFORMATION, true), + entry(TEST_METHOD, false), + entry(PII, false), + entry(PURITY, false), + entry(IMAGE, true), + entry(OCR, true), + entry(SIGNATURE, true), + entry(LOGO, true), + entry(FORMULA, true), + entry(DOSSIER_REDACTIONS, false), + entry(IMPORTED_REDACTION, false)); private static final Map recommendationTypeMap = Map.ofEntries(entry(VERTEBRATE, false), - entry(ADDRESS, false), - entry(AUTHOR, false), - entry(SPONSOR, false), - entry(NO_REDACTION_INDICATOR, false), - entry(REDACTION_INDICATOR, false), - entry(HINT_ONLY, false), - entry(MUST_REDACT, false), - entry(PUBLISHED_INFORMATION, false), - entry(TEST_METHOD, false), - entry(PII, false), - entry(PURITY, false), - entry(IMAGE, false), - entry(OCR, false), - entry(FORMULA, false), - entry(SIGNATURE, false), - entry(LOGO, false), - entry(DOSSIER_REDACTIONS, false), - entry(IMPORTED_REDACTION, false)); + entry(ADDRESS, false), + entry(AUTHOR, false), + entry(SPONSOR, false), + entry(NO_REDACTION_INDICATOR, false), + entry(REDACTION_INDICATOR, false), + entry(HINT_ONLY, false), + entry(MUST_REDACT, false), + entry(PUBLISHED_INFORMATION, false), + entry(TEST_METHOD, false), + entry(PII, false), + entry(PURITY, false), + entry(IMAGE, false), + entry(OCR, false), + entry(FORMULA, false), + entry(SIGNATURE, false), + entry(LOGO, false), + entry(DOSSIER_REDACTIONS, false), + entry(IMPORTED_REDACTION, false)); private static final Map rankTypeMap = Map.ofEntries(entry(PURITY, 155), - entry(PII, 150), - entry(ADDRESS, 140), - entry(AUTHOR, 130), - entry(SPONSOR, 120), - entry(VERTEBRATE, 110), - entry(MUST_REDACT, 100), - entry(REDACTION_INDICATOR, 90), - entry(NO_REDACTION_INDICATOR, 80), - entry(PUBLISHED_INFORMATION, 70), - entry(TEST_METHOD, 60), - entry(HINT_ONLY, 50), - entry(IMAGE, 30), - entry(OCR, 29), - entry(LOGO, 28), - entry(SIGNATURE, 27), - entry(FORMULA, 26), - entry(DOSSIER_REDACTIONS, 200), - entry(IMPORTED_REDACTION, 200)); + entry(PII, 150), + entry(ADDRESS, 140), + entry(AUTHOR, 130), + entry(SPONSOR, 120), + entry(VERTEBRATE, 110), + entry(MUST_REDACT, 100), + entry(REDACTION_INDICATOR, 90), + entry(NO_REDACTION_INDICATOR, 80), + entry(PUBLISHED_INFORMATION, 70), + entry(TEST_METHOD, 60), + entry(HINT_ONLY, 50), + entry(IMAGE, 30), + entry(OCR, 29), + entry(LOGO, 28), + entry(SIGNATURE, 27), + entry(FORMULA, 26), + entry(DOSSIER_REDACTIONS, 200), + entry(IMPORTED_REDACTION, 200)); private final Colors colors = new Colors(); @Autowired @@ -273,15 +273,15 @@ public class RulesTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) - .build())); + .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS)) + .build())); mockDictionaryCalls(null); mockDictionaryCalls(0L); @@ -372,10 +372,10 @@ public class RulesTest { log.warn("WARNING: {} files from {} failed", failedFiles.size(), fileSize); for (String fileName : failedFiles.keySet()) { log.warn(" - '{}' failed with Error: {} See line {} in {}", - fileName, - failedFiles.get(fileName), - failedFiles.get(fileName).getStackTrace()[0].getLineNumber(), - failedFiles.get(fileName).getStackTrace()[0].getClassName()); + fileName, + failedFiles.get(fileName), + failedFiles.get(fileName).getStackTrace()[0].getLineNumber(), + failedFiles.get(fileName).getStackTrace()[0].getClassName()); } } @@ -442,9 +442,13 @@ public class RulesTest { assertThat(entityLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion()); assertThat(entityLog.getEntityLogEntry() - .stream().filter(r -> !r.getEntryType().equals(EntryType.FALSE_POSITIVE)).filter(r -> !r.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) - .collect(Collectors.toSet()) - .size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size()); + .stream() + .filter(r -> !r.getEntryType().equals(EntryType.FALSE_POSITIVE)) + .filter(r -> !r.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) + .collect(Collectors.toSet()).size()).isEqualTo(savedRedactionLog.getRedactionLogEntry() + .stream() + .filter(r -> !r.isFalsePositive()) + .collect(Collectors.toSet()).size()); assertThat(entityLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size()); for (EntityLogLegalBasis redactionLegalBasis : entityLog.getLegalBasis()) { @@ -458,7 +462,10 @@ public class RulesTest { } for (EntityLogEntry redactionLogEntry : entityLog.getEntityLogEntry()) { - var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst(); + var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry() + .stream() + .filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())) + .findFirst(); assertThat(savedRedactionLogEntry).isPresent(); assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId()); assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType()); @@ -473,7 +480,8 @@ public class RulesTest { assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)); assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection()); assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor()); - assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId().get(0)); + assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId() + .get(0)); assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore()); assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter()); assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset()); @@ -485,8 +493,7 @@ public class RulesTest { assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded()); for (Position position : redactionLogEntry.getPositions()) { - var savedRectangle = savedRedactionLogEntry.get() - .getPositions() + var savedRectangle = savedRedactionLogEntry.get().getPositions() .stream() .filter(r -> r.getPage() == position.getPageNumber()) .filter(r -> r.getTopLeft().getX() == position.x()) @@ -498,17 +505,16 @@ public class RulesTest { } for (Change change : redactionLogEntry.getChanges()) { - var savedChange = savedRedactionLogEntry.get() - .getChanges() + var savedChange = savedRedactionLogEntry.get().getChanges() .stream() - .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber()).filter(c -> c.getType().name().equals(change.getType().name())) + .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber()) + .filter(c -> c.getType().name().equals(change.getType().name())) .findFirst(); assertThat(savedChange).isPresent(); } for (ManualChange manualChange : redactionLogEntry.getManualChanges()) { - var savedManualChange = savedRedactionLogEntry.get() - .getManualChanges() + var savedManualChange = savedRedactionLogEntry.get().getManualChanges() .stream() .filter(m -> m.getManualRedactionType().name().equals(manualChange.getManualRedactionType().name())) .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId())) @@ -517,7 +523,8 @@ public class RulesTest { assertThat(savedManualChange).isPresent(); } - assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(Engine[]::new)); + assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines() + .toArray(Engine[]::new)); assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference()); assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections()); @@ -622,8 +629,8 @@ public class RulesTest { ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), - responseJson.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), + responseJson.getInputStream()); } @@ -647,8 +654,8 @@ public class RulesTest { .build(); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), - new ClassPathResource("files/cv_service_empty_response.json").getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), + new ClassPathResource("files/cv_service_empty_response.json").getInputStream()); storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream); return request; @@ -659,45 +666,102 @@ public class RulesTest { private void loadDictionaryForTest() { dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_author.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/vertebrate.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/CBI_address.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/hint_only.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/must_redact.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/published_information.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/test_method.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(PII, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/PII.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/purity.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(OCR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/empty.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); falsePositive.computeIfAbsent(PII, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet())); + .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); } @@ -736,12 +800,12 @@ public class RulesTest { when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false)); when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, - false)); + false)); when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false)); when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false)); when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, - false)); + false)); when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false)); when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false)); when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false)); @@ -809,7 +873,10 @@ public class RulesTest { protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType) { return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, - AnalyzeRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build())); + AnalyzeRequest.builder() + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java index f6f6a5f8..64a43917 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.annotate; - import java.util.List; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index be02c70e..0618888d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -79,8 +79,7 @@ public class AnnotationService { try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { pdDocument.save(byteArrayOutputStream); - return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()) - .build(); + return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build(); } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java index b6f4b2d9..d5fa1c4f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java @@ -13,10 +13,7 @@ public class TextEntityTest { @Test public void testMatchedRule() { - PrecursorEntity entity = PrecursorEntity.builder() - .type("PII") - .entityType(EntityType.ENTITY) - .build(); + PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build(); entity.skip("CBI.1.0", ""); entity.skip("CBI.2.0", ""); entity.skip("CBI.3.0", ""); @@ -27,14 +24,10 @@ public class TextEntityTest { } - @Test public void testMatchedRuleWithNonsense() { - PrecursorEntity entity = PrecursorEntity.builder() - .type("PII") - .entityType(EntityType.ENTITY) - .build(); + PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build(); assertThrows(IllegalArgumentException.class, () -> { entity.skip("", ""); }); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index 739eb54c..f952c80b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -49,19 +49,26 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); } + @Test public void assertEntitiesAreDuplicatedWithTheirTableCell() { Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf"); - List entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document).toList(); + List entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document) + .toList(); assertEquals(3, entities.size()); - assertEquals(1, entities.stream().distinct().count()); + assertEquals(1, + entities.stream() + .distinct() + .count()); assertEquals(2, entities.get(0).getDuplicateTextRanges().size()); var node = entities.get(0).getDeepestFullyContainingNode(); assertTrue(node.getTextRange().contains(entities.get(0).getTextRange())); - assertTrue(entities.get(0).getDuplicateTextRanges().stream().allMatch(tr -> node.getTextRange().contains(tr))); + assertTrue(entities.get(0).getDuplicateTextRanges() + .stream() + .allMatch(tr -> node.getTextRange().contains(tr))); } @@ -84,8 +91,10 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr Document document = buildGraph("files/new/crafted document.pdf"); String type = "CBI_author"; - assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); - assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document).isPresent()); + assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document) + .isPresent()); + assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document) + .isPresent()); assertEquals(1, document.getEntities().size()); verify(kieSession, times(1)).insert(any(TextEntity.class)); } @@ -114,7 +123,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assertEquals("’s Donut ←", textEntity.getTextAfter()); assertEquals(searchTerm, textEntity.getValue()); assertEquals("Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) ", - textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); + textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(3, textEntity.getIntersectingNodes().size()); assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(Paragraph.class, textEntity.getDeepestFullyContainingNode()); @@ -181,11 +190,18 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr .allEntriesInOrder()// .filter(entry -> entry.getType().equals(NodeType.TABLE))// .map(DocumentTree.Entry::getNode)// - .findFirst().orElseThrow(); + .findFirst() + .orElseThrow(); assertEquals(5, table.getNumberOfCols()); assertEquals(4, table.getNumberOfRows()); - assertEquals(5, table.streamHeaders().toList().size()); - CharSequence firstHeader = table.streamHeadersForCell(1, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).findFirst().orElseThrow(); + assertEquals(5, + table.streamHeaders() + .toList().size()); + CharSequence firstHeader = table.streamHeadersForCell(1, 1) + .map(TableCell::getTextBlock) + .map(TextBlock::getSearchText) + .findFirst() + .orElseThrow(); assertEquals("Author(s)", firstHeader.toString().stripTrailing()); } @@ -207,17 +223,23 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr public void assertTableStructureMetolachlor() { Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); - Table table = (Table) document.getDocumentTree() - .allEntriesInOrder() - .filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22)) + Table table = (Table) document.getDocumentTree().allEntriesInOrder() + .filter(entry -> entry.getNode().getPages() + .stream() + .anyMatch(page -> page.getNumber() == 22)) .filter(entry -> entry.getType().equals(NodeType.TABLE)) .map(DocumentTree.Entry::getNode) .findFirst() .orElseThrow(); assertEquals(5, table.getNumberOfCols()); assertEquals(14, table.getNumberOfRows()); - assertEquals(10, table.streamHeaders().toList().size()); - List twoHeaders = table.streamHeadersForCell(2, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).toList(); + assertEquals(10, + table.streamHeaders() + .toList().size()); + List twoHeaders = table.streamHeadersForCell(2, 1) + .map(TableCell::getTextBlock) + .map(TextBlock::getSearchText) + .toList(); assertEquals(2, twoHeaders.size()); assertEquals("Component of residue definition: S-Metolachlor", twoHeaders.get(0).stripTrailing()); assertEquals("Method type", twoHeaders.get(1).stripTrailing()); @@ -233,12 +255,13 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assertEquals("except Cranberry; Vegetable, ", textEntity.getTextBefore()); assertEquals(", Group 9;", textEntity.getTextAfter()); - assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ", - textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); + assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ", textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(searchTerm, textEntity.getValue()); assertEquals(3, textEntity.getIntersectingNodes().size()); assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); - assertTrue(textEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 10)); + assertTrue(textEntity.getPages() + .stream() + .allMatch(pageNode -> pageNode.getNumber() == 10)); assertInstanceOf(Paragraph.class, textEntity.getDeepestFullyContainingNode()); assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity); @@ -262,11 +285,13 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assertEquals("2.6.1 Summary of ", textEntity.getTextBefore()); assertEquals(" and excretion in", textEntity.getTextAfter()); assertEquals("2.6.1 Summary of absorption, distribution, metabolism and excretion in mammals ", - textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); + textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); assertEquals(searchTerm, textEntity.getValue()); assertEquals(3, textEntity.getIntersectingNodes().size()); assertEquals(4, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); - assertTrue(textEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 33)); + assertTrue(textEntity.getPages() + .stream() + .allMatch(pageNode -> pageNode.getNumber() == 33)); assertInstanceOf(Headline.class, textEntity.getDeepestFullyContainingNode()); assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity); @@ -285,8 +310,10 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assertEquals(searchTerm, textEntity.getValue()); assertEquals(4, textEntity.getIntersectingNodes().size()); assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ", - textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); - assertTrue(textEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54)); + textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); + assertTrue(textEntity.getPages() + .stream() + .allMatch(pageNode -> pageNode.getNumber() == 54)); assertEquals(26, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(TableCell.class, textEntity.getDeepestFullyContainingNode()); @@ -298,7 +325,8 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr // this might fail, if an entity with the same name exists twice in the deepest containing node private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, TextEntity textEntity) { - List paragraphStart = textEntity.getIntersectingNodes().stream()// + List paragraphStart = textEntity.getIntersectingNodes() + .stream()// .map(SemanticNode::getTextBlock)// .map(textBlock -> textBlock.indexOf(searchTerm, textEntity.getDeepestFullyContainingNode().getTextRange().start()))// .toList(); @@ -316,14 +344,23 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr TextRange textRange = new TextRange(start, start + searchTerm.length()); TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document); entityCreationService.addEntityToGraph(textEntity, document); - Page pageNode = document.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); + Page pageNode = document.getPages() + .stream() + .filter(page -> page.getNumber() == pageNumber) + .findFirst() + .orElseThrow(); assertEquals(textEntity.getValue(), searchTerm); assertTrue(pageNode.getEntities().contains(textEntity)); - assertTrue(document.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(textEntity))); + assertTrue(document.getPages() + .stream() + .filter(page -> page != pageNode) + .noneMatch(page -> page.getEntities().contains(textEntity))); assertTrue(textEntity.getPages().contains(pageNode)); assertSameOffsetInAllIntersectingNodes(searchTerm, textEntity); - assertTrue(textEntity.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(textEntity))); + assertTrue(textEntity.getIntersectingNodes() + .stream() + .allMatch(node -> node.getEntities().contains(textEntity))); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index e3e6a69c..2dd69b86 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -112,15 +112,15 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -155,11 +155,11 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration long droolsStart = System.currentTimeMillis(); List fileAttributes = entityDroolsExecutionService.executeRules(kieContainer, - document, - dictionary, - Collections.emptyList(), - new ManualRedactions(), - new NerEntities()); + document, + dictionary, + Collections.emptyList(), + new ManualRedactions(), + new NerEntities()); System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart); System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); @@ -181,7 +181,8 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration float durationMillis = ((float) (System.currentTimeMillis() - start)); System.out.printf("%d calls of buildTextBlock() on document took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); - Section section = document.getMainSections().get(8); + Section section = document.getMainSections() + .get(8); start = System.currentTimeMillis(); for (int i = 0; i < n; i++) { section.getTextBlock(); @@ -245,11 +246,19 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns); System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size()); for (TextEntity entity : document.getEntities()) { - var foundEntity = foundEntities.stream().filter(f -> f.getId().equals(entity.getId())).findFirst().get(); + var foundEntity = foundEntities.stream() + .filter(f -> f.getId().equals(entity.getId())) + .findFirst() + .get(); assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange())); } - assert document.getEntities().stream().mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size(); - assert foundEntities.stream().map(TextEntity::getId).distinct().count() == document.getEntities().size(); + assert document.getEntities() + .stream() + .mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size(); + assert foundEntities.stream() + .map(TextEntity::getId) + .distinct() + .count() == document.getEntities().size(); drawAllEntities(filename, document); } @@ -265,9 +274,11 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration .stream() .filter(entityNode -> !entityNode.removed()) .filter(TextEntity::applied) - .flatMap(entityNode -> entityNode.getPositionsOnPagePerPage().stream()) + .flatMap(entityNode -> entityNode.getPositionsOnPagePerPage() + .stream()) .filter(entityPosition -> entityPosition.getPage().equals(page)) - .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) + .flatMap(entityPosition -> entityPosition.getRectanglePerLine() + .stream()) .toList(); PdfVisualisationUtility.Options options = PdfVisualisationUtility.Options.builder().strokeColor(Color.BLACK).stroke(true).build(); @@ -279,9 +290,11 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration .stream() .filter(entityNode -> !entityNode.removed()) .filter(entityNode -> !entityNode.applied()) - .flatMap(entityNode -> entityNode.getPositionsOnPagePerPage().stream()) + .flatMap(entityNode -> entityNode.getPositionsOnPagePerPage() + .stream()) .filter(entityPosition -> entityPosition.getPage().equals(page)) - .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) + .flatMap(entityPosition -> entityPosition.getRectanglePerLine() + .stream()) .toList(); PdfVisualisationUtility.Options options = PdfVisualisationUtility.Options.builder().strokeColor(Color.BLUE).stroke(true).build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java index cded6d0f..d99ff66b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java @@ -15,8 +15,17 @@ public class DocumentTableIntegrationTest extends BuildDocumentIntegrationTest { public void testAllTableCellAccessesCorrect() { Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06"); - Page pageFive = document.getPages().stream().filter(pageNode -> pageNode.getNumber() == 5).findFirst().get(); - Table table = pageFive.getMainBody().stream().filter(semanticNode -> semanticNode instanceof Table).map(semanticNode -> (Table) semanticNode).findFirst().get(); + Page pageFive = document.getPages() + .stream() + .filter(pageNode -> pageNode.getNumber() == 5) + .findFirst() + .get(); + Table table = pageFive.getMainBody() + .stream() + .filter(semanticNode -> semanticNode instanceof Table) + .map(semanticNode -> (Table) semanticNode) + .findFirst() + .get(); for (int row = 0; row < table.getNumberOfRows(); row++) { for (int col = 0; col < table.getNumberOfCols(); col++) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java index 024660e8..45282452 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java @@ -83,8 +83,8 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati PdfVisualisationUtility.drawDocumentGraph(pdDocument, document); PdfVisualisationUtility.drawTextBlock(pdDocument, - textBlock, - PdfVisualisationUtility.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build()); + textBlock, + PdfVisualisationUtility.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build()); File outputFile = new File(tmpFileName); pdDocument.save(outputFile); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java index 8fc4834f..8b090cb9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java @@ -27,7 +27,8 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest { SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - List entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document).toList(); + List entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document) + .toList(); assertEquals(2, entities.size()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java index 27f29647..48d612a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java @@ -79,6 +79,7 @@ class DroolsSyntaxValidationServiceTest { assertTrue(droolsSyntaxValidation.isCompiled()); } + @Test @SneakyThrows void testRulesWithAddedImports() { @@ -96,6 +97,7 @@ class DroolsSyntaxValidationServiceTest { assertTrue(droolsSyntaxValidation.isCompiled()); } + @Test @SneakyThrows void testDocumineRules() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java index 6e7af29f..b6ead597 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java @@ -26,7 +26,9 @@ public class DroolsUpToDateTest { public void assertAllRuleFilesAreUpToDate() { Path droolsPath = new ClassPathResource("drools").getFile().toPath(); - Files.walk(droolsPath).filter(DroolsUpToDateTest::isEntityRuleFile).forEach(this::validateFile); + Files.walk(droolsPath) + .filter(DroolsUpToDateTest::isEntityRuleFile) + .forEach(this::validateFile); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index c483a49a..346af17c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -127,15 +127,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -155,29 +155,39 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { String testEntityValue1 = "Desiree"; String testEntityValue2 = "Melanie"; EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); - assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count()); + assertEquals(2, + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue1)) + .count()); + assertEquals(2, + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue2)) + .count()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; - entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); + entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document) + .findFirst() + .get(); String idToResize = redactionLog.getEntityLogEntry() .stream() .filter(entry -> entry.getValue().equals(testEntityValue1)) .max(Comparator.comparingInt(EntityLogEntry::getStartOffset)) - .get() - .getId(); + .get().getId(); ManualRedactions manualRedactions = new ManualRedactions(); - manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder() - .annotationId(idToResize) - .value(expandedEntityKeyword) - .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(), - Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build())) - .addToAllDossiers(false) - .updateDictionary(false) - .requestDate(OffsetDateTime.now()) - .build()); + manualRedactions.getResizeRedactions() + .add(ManualResizeRedaction.builder() + .annotationId(idToResize) + .value(expandedEntityKeyword) + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(), + Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build())) + .addToAllDossiers(false) + .updateDictionary(false) + .requestDate(OffsetDateTime.now()) + .build()); request.setManualRedactions(manualRedactions); analyzeService.reanalyze(request); @@ -188,21 +198,32 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) { fileOutputStream.write(annotateResponse.getDocument()); } - EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get(); + EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(expandedEntityKeyword)) + .findFirst() + .get(); assertEquals(idToResize, resizedEntry.getId()); - assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count()); assertEquals(1, - redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count()); + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue1)) + .count()); + assertEquals(1, + redactionLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)) + .count()); } private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(), - (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(), - (float) rectangle2D.getWidth(), - -(float) rectangle2D.getHeight(), - pageNumber); + (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(), + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); } @@ -219,10 +240,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() - .annotationId("675eba69b0c2917de55462c817adaa05") - .fileId("fileId") - .legalBasis("Something") - .build())); + .annotationId("675eba69b0c2917de55462c817adaa05") + .fileId("fileId") + .legalBasis("Something") + .build())); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); manualRedactionEntry.setAnnotationId(manualAddId); @@ -232,7 +253,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setReason("Manual Redaction"); manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), - Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); + Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); AnalyzeRequest request = uploadFileToStorage(pdfFile); request.setManualRedactions(manualRedactions); @@ -242,11 +263,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() - .annotationId("675eba69b0c2917de55462c817adaa05") - .fileId("fileId") - .legalBasis("Manual Legal Basis Change") - .requestDate(OffsetDateTime.now()) - .build()))); + .annotationId("675eba69b0c2917de55462c817adaa05") + .fileId("fileId") + .legalBasis("Manual Legal Basis Change") + .requestDate(OffsetDateTime.now()) + .build()))); analyzeService.reanalyze(request); @@ -295,7 +316,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } long end = System.currentTimeMillis(); - var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny(); + var optionalEntry = redactionLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findAny(); assertTrue(optionalEntry.isPresent()); assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell System.out.println("duration: " + (end - start)); @@ -345,9 +369,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); assertFalse(redactionLog2.getEntityLogEntry() - .stream() - .filter(entry -> entry.getType().equals("published_information")) - .anyMatch(entry -> entry.getValue().equals("Oxford University Press"))); + .stream() + .filter(entry -> entry.getType().equals("published_information")) + .anyMatch(entry -> entry.getValue().equals("Oxford University Press"))); var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry() .stream() @@ -381,15 +405,15 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { String annotationId = "testAnnotationId"; manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder() - .annotationId(annotationId) - .requestDate(OffsetDateTime.now()) - .type("manual") - .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") - .positions(List.of(// - new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // - new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // - new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // - .build())); + .annotationId(annotationId) + .requestDate(OffsetDateTime.now()) + .type("manual") + .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") + .positions(List.of(// + new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // + new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // + new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // + .build())); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(annotationId) .requestDate(OffsetDateTime.now()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index 860f4d1b..5e5f4ded 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -32,18 +32,31 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualResizeRedactionTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); Set biggerEntities = entityCreationService.byString("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, document) .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); - TextEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); + TextEntity biggerEntity = biggerEntities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(initialId) .value(biggerEntity.getValue()) - .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0))) + .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage() + .get(0))) .requestDate(OffsetDateTime.now()) .updateDictionary(false) .build(); @@ -55,8 +68,13 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty()); assertEquals(biggerEntity.getPages(), entity.getPages()); assertEquals(biggerEntity.getValue(), entity.getValue()); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); - assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage().get(0).getRectanglePerLine(), entity.getPositionsOnPagePerPage().get(0).getRectanglePerLine()); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); + assertRectanglesAlmostEqual(biggerEntity.getPositionsOnPagePerPage() + .get(0).getRectanglePerLine(), + entity.getPositionsOnPagePerPage() + .get(0).getRectanglePerLine()); assertTrue(entity.resized()); } @@ -65,11 +83,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualForceRedactionTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build(); doAnalysis(document, List.of(manualForceRedaction)); @@ -78,8 +103,12 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertFalse(entity.getIntersectingNodes().isEmpty()); assertEquals(1, entity.getPages().size()); assertEquals("David Ksenia", entity.getValue()); - assertEquals("Something", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); + assertEquals("Something", + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); assertFalse(entity.removed()); assertTrue(entity.hasManualChanges()); assertTrue(entity.applied()); @@ -90,17 +119,26 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualIDRemovalTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build(); doAnalysis(document, List.of(idRemoval)); assertEquals("David Ksenia", entity.getValue()); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); assertTrue(entity.ignored()); } @@ -109,11 +147,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { public void manualIDRemovalButAlsoForceRedactionTest() { Document document = buildGraph("files/new/crafted document"); - Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet()); + Set entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document) + .collect(Collectors.toUnmodifiableSet()); - TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get(); + TextEntity entity = entities.stream() + .filter(e -> e.getPages() + .stream() + .anyMatch(p -> p.getNumber() == 1)) + .findFirst() + .get(); - String initialId = entity.getPositionsOnPagePerPage().get(0).getId(); + String initialId = entity.getPositionsOnPagePerPage() + .get(0).getId(); IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build(); ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build(); @@ -123,7 +168,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertFalse(entity.getIntersectingNodes().isEmpty()); assertEquals(1, entity.getPages().size()); assertEquals("David Ksenia", entity.getValue()); - assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); + assertEquals(initialId, + entity.getPositionsOnPagePerPage() + .get(0).getId()); assertFalse(entity.removed()); assertFalse(entity.ignored()); } @@ -131,7 +178,9 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { private void assertRectanglesAlmostEqual(Collection rects1, Collection rects2) { - if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) { + if (rects1.stream() + .allMatch(rect1 -> rects2.stream() + .anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) { return; } // use this for nice formatting of error message @@ -143,15 +192,18 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { double tolerance = 1e-1; return Math.abs(r1.getX() - r2.getX()) < tolerance &&// - Math.abs(r1.getY() - r2.getY()) < tolerance &&// - Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&// - Math.abs(r1.getHeight() - r2.getHeight()) < tolerance; + Math.abs(r1.getY() - r2.getY()) < tolerance &&// + Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&// + Math.abs(r1.getHeight() - r2.getHeight()) < tolerance; } private static List toAnnotationRectangles(PositionOnPage positionsOnPage) { - return positionsOnPage.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())).toList(); + return positionsOnPage.getRectanglePerLine() + .stream() + .map(rectangle2D -> toAnnotationRectangle(rectangle2D, positionsOnPage.getPage().getNumber())) + .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index e5761ebe..0ff00218 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -43,7 +43,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { OffsetDateTime start = OffsetDateTime.now(); String reason = "whatever"; Document document = buildGraphNoImages("files/new/crafted document.pdf"); - List entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document).peek(e -> e.apply("T.0.0", reason)).toList(); + List entities = entityCreationService.byString("David Ksenia", "test", EntityType.ENTITY, document) + .peek(e -> e.apply("T.0.0", reason)) + .toList(); assertFalse(entities.isEmpty()); TextEntity entity = entities.get(0); assertTrue(entity.active()); @@ -52,7 +54,8 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.resized()); assertFalse(entity.ignored()); assertEquals("n-a", entity.getMatchedRule().getLegalBasis()); - String annotationId = entity.getPositionsOnPagePerPage().get(0).getId(); + String annotationId = entity.getPositionsOnPagePerPage() + .get(0).getId(); // remove first IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build(); @@ -73,7 +76,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.ignored()); assertFalse(entity.removed()); assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions()); - assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals("coolio", + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); // remove again IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build(); @@ -93,7 +98,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertTrue(entity.ignored()); assertFalse(entity.applied()); assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override", - entity.buildReasonWithManualChangeDescriptions()); + entity.buildReasonWithManualChangeDescriptions()); String legalBasis = "Yeah"; String section = "Some random section!"; @@ -110,10 +115,16 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertTrue(entity.ignored()); assertFalse(entity.applied()); assertEquals(reason + ", removed by manual override, forced by manual override, forced by manual override, removed by manual override, legal basis was manually changed", - entity.buildReasonWithManualChangeDescriptions()); - assertEquals(value, entity.getManualOverwrite().getValue().orElse(entity.getValue())); - assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); - assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())); + entity.buildReasonWithManualChangeDescriptions()); + assertEquals(value, + entity.getManualOverwrite().getValue() + .orElse(entity.getValue())); + assertEquals(legalBasis, + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals(section, + entity.getManualOverwrite().getSection() + .orElse(entity.getDeepestFullyContainingNode().toString())); ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder() .type("type") @@ -122,9 +133,13 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { .legalBasis("") .build(); entity.getManualOverwrite().addChange(imageRecategorizationRequest); - assertTrue(entity.getManualOverwrite().getRecategorized().isPresent()); - assertTrue(entity.getManualOverwrite().getRecategorized().get()); - assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type())); + assertTrue(entity.getManualOverwrite().getRecategorized() + .isPresent()); + assertTrue(entity.getManualOverwrite().getRecategorized() + .get()); + assertEquals("type", + entity.getManualOverwrite().getType() + .orElse(entity.type())); } @@ -134,7 +149,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { OffsetDateTime start = OffsetDateTime.now(); String reason = "whatever"; Document document = buildGraphNoImages("files/new/crafted document.pdf"); - List entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document).peek(e -> e.apply("T.0.0", reason)).toList(); + List entities = entityCreationService.byString("David Ksenia", "test", EntityType.HINT, document) + .peek(e -> e.apply("T.0.0", reason)) + .toList(); assertFalse(entities.isEmpty()); TextEntity entity = entities.get(0); assertTrue(entity.active()); @@ -143,7 +160,8 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.resized()); assertFalse(entity.ignored()); assertEquals("n-a", entity.getMatchedRule().getLegalBasis()); - String annotationId = entity.getPositionsOnPagePerPage().get(0).getId(); + String annotationId = entity.getPositionsOnPagePerPage() + .get(0).getId(); // remove first IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build(); @@ -164,7 +182,9 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { assertFalse(entity.ignored()); assertFalse(entity.removed()); assertEquals(reason + ", removed by manual override, forced by manual override", entity.buildReasonWithManualChangeDescriptions()); - assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis())); + assertEquals("coolio", + entity.getManualOverwrite().getLegalBasis() + .orElse(entity.getMatchedRule().getLegalBasis())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index b10c8a6a..94f0823f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -122,17 +122,20 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), - document, - TEST_DOSSIER_TEMPLATE_ID); + List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder() + .entriesToAdd(Set.of( + manualRedactionEntry)) + .build(), + document, + TEST_DOSSIER_TEMPLATE_ID); assertEquals(1, notFoundManualEntities.size()); assertTrue(document.getEntities().isEmpty()); List redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(), - document, - notFoundManualEntities, - new DictionaryVersion(), - 0L).getEntityLogEntry(); + document, + notFoundManualEntities, + new DictionaryVersion(), + 0L).getEntityLogEntry(); assertEquals(1, redactionLogEntries.size()); assertEquals(value, redactionLogEntries.get(0).getValue()); @@ -146,7 +149,8 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { Document document = buildGraph("files/new/VV-919901.pdf"); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList(); + List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document) + .toList(); assertFalse(tempEntities.isEmpty()); var tempEntity = tempEntities.get(0); List positions = tempEntity.getPositionsOnPagePerPage() @@ -172,21 +176,28 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { tempEntity.removeFromGraph(); assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), - document, - TEST_DOSSIER_TEMPLATE_ID); + List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder() + .entriesToAdd(Set.of( + manualRedactionEntry)) + .build(), + document, + TEST_DOSSIER_TEMPLATE_ID); assertTrue(notFoundManualEntities.isEmpty()); assertEquals(1, document.getEntities().size()); - return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get()); + return new DocumentAndEntity(document, + document.getEntities() + .stream() + .findFirst() + .get()); } public static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) { return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), - (float) rectangle2D.getWidth(), - -(float) rectangle2D.getHeight(), - pageNumber); + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java index d5215b4a..35f035fd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java @@ -22,7 +22,6 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest public static final String FILE_NAME = "test-file"; - @Test @SneakyThrows public void testFile() { @@ -41,13 +40,15 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest redactionServiceSettings.setNerServiceEnabled(false); } - AnalyzeRequest ar = AnalyzeRequest.builder().fileId(FILE_ID).dossierId(DOSSIER_ID) - .analysisNumber(1).dossierTemplateId(DOSSIER_TEMPLATE_ID) + AnalyzeRequest ar = AnalyzeRequest.builder() + .fileId(FILE_ID) + .dossierId(DOSSIER_ID) + .analysisNumber(1) + .dossierTemplateId(DOSSIER_TEMPLATE_ID) .lastProcessed(OffsetDateTime.now()) .fileAttributes(List.of()) .build(); - // try { // var redactionLog = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".redaction-log.json").getInputStream(); // } catch (Exception e) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index 4f8ddfe5..8cc15736 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -159,14 +159,19 @@ public class LiveDataIntegrationTest { when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>()); when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), - Colors.class)); + Colors.class)); when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> { String typeName = answer.getArgument(0); - var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst(); + var found = types.stream() + .filter(t -> t.getType().equalsIgnoreCase(typeName)) + .findFirst(); if (found.isPresent()) { - var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get(); + var type = types.stream() + .filter(t -> t.getType().equalsIgnoreCase(typeName)) + .findFirst() + .get(); type.setEntries(getEntries(typeName, type.getTypeId())); type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId())); @@ -188,13 +193,20 @@ public class LiveDataIntegrationTest { when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> { String typeName = answer.getArgument(0); - var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst(); + var found = types.stream() + .filter(t -> t.getType().equalsIgnoreCase(typeName)) + .findFirst(); if (found.isPresent()) { - var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get(); + var type = types.stream() + .filter(t -> t.getType().equalsIgnoreCase(typeName)) + .findFirst() + .get(); if (deltaTypeName.equalsIgnoreCase(typeName)) { - type.setEntries(values.stream().map(l -> new DictionaryEntry(0, l, version, false, type.getTypeId())).collect(Collectors.toList())); + type.setEntries(values.stream() + .map(l -> new DictionaryEntry(0, l, version, false, type.getTypeId())) + .collect(Collectors.toList())); } else { type.setEntries(new ArrayList<>()); } @@ -226,7 +238,9 @@ public class LiveDataIntegrationTest { if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) { List lines = IOUtils.readLines(resource.getInputStream()); - return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList()); + return lines.stream() + .map(l -> new DictionaryEntry(0, l, 0L, false, typeId)) + .collect(Collectors.toList()); } } @@ -242,7 +256,9 @@ public class LiveDataIntegrationTest { if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) { List lines = IOUtils.readLines(resource.getInputStream()); - return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList()); + return lines.stream() + .map(l -> new DictionaryEntry(0, l, 0L, false, typeId)) + .collect(Collectors.toList()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index c507b417..c86b2da8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -68,13 +68,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { Document document = buildGraphNoImages(filePath); List entityRecognitionEntities = validateAndCombine(parseNerEntities(nerEntitiesFilePath), document); assertFalse(entityRecognitionEntities.isEmpty()); - assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.textRange().start() < entity.textRange().end())); + assertTrue(entityRecognitionEntities.stream() + .allMatch(entity -> entity.textRange().start() < entity.textRange().end())); ClassPathResource resource = new ClassPathResource(filePath); try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { - Stream unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) - .getNerEntityList() + Stream unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document).getNerEntityList() .stream() .filter(e -> !e.type().equals("CBI_author")); List redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts) @@ -83,12 +83,15 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { .map(Optional::get) .toList(); redactionEntities.stream() - .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) + .collect(Collectors.groupingBy(e -> e.getPages() + .stream() + .findFirst() + .get().getNumber())) .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber, - getPositionsFromEntityOfType("CBI_author", entities), - getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), - getPositionsFromEntityOfType("CBI_address", entities), - pdDocument)); + getPositionsFromEntityOfType("CBI_author", entities), + getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), + getPositionsFromEntityOfType("CBI_address", entities), + pdDocument)); File outputFile = new File("/tmp/nerEntities.pdf"); pdDocument.save(outputFile); @@ -109,7 +112,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { log.info("Parsed NerEntitiesModel"); NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document); log.info("Validated and mapped"); - List nerEntityBoundaries = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList(); + List nerEntityBoundaries = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) + .toList(); log.info("Combined to CBI_address"); List cbiAddressEntities = nerEntityBoundaries.stream() .map(b -> entityCreationService.byTextRange(b, "CBI_address", EntityType.RECOMMENDATION, document)) @@ -117,25 +121,28 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { .map(Optional::get) .toList(); assertFalse(cbiAddressEntities.isEmpty()); - assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end())); + assertTrue(cbiAddressEntities.stream() + .allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end())); ClassPathResource resource = new ClassPathResource(filePath); try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { - List validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) - .getNerEntityList() + List validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document).getNerEntityList() .stream() .map(e -> entityCreationService.byTextRange(e.textRange(), e.type(), EntityType.ENTITY, document)) .filter(Optional::isPresent) .map(Optional::get) .toList(); Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream()) - .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) + .collect(Collectors.groupingBy(e -> e.getPages() + .stream() + .findFirst() + .get().getNumber())) .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber, - getPositionsFromEntityOfType("CBI_author", entities), - getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), - getPositionsFromEntityOfType("CBI_address", entities), - pdDocument)); + getPositionsFromEntityOfType("CBI_author", entities), + getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities), + getPositionsFromEntityOfType("CBI_address", entities), + pdDocument)); File outputFile = new File("/tmp/nerEntities.pdf"); pdDocument.save(outputFile); @@ -147,11 +154,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { NerEntities nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document); - List cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList(); + List cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author") + .toList(); Stream cbiAddress = NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> new NerEntities.NerEntity(document.getTextBlock().subSequence(boundary).toString(), boundary, "CBI_address")); - return Stream.concat(cbiAuthors.stream(), cbiAddress).toList(); + return Stream.concat(cbiAuthors.stream(), cbiAddress) + .toList(); } @@ -167,14 +176,17 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { private List getPositionsFromEntityOfType(String type, List entities) { - return getPositionsFromEntities(entities.stream().filter(e -> e.type().equals(type))); + return getPositionsFromEntities(entities.stream() + .filter(e -> e.type().equals(type))); } private List getPositionsFromEntityNotOfType(List types, List entities) { - return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.type().equals(type)))); + return getPositionsFromEntities(entities.stream() + .filter(e -> types.stream() + .noneMatch(type -> e.type().equals(type)))); } @@ -188,13 +200,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, cbiAuthorRects, PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.blue).build()); PdfVisualisationUtility.drawRectangle2DList(pdDocument, - pageNumber, - addressPartsRects, - PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.MAGENTA).build()); + pageNumber, + addressPartsRects, + PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.MAGENTA).build()); PdfVisualisationUtility.drawRectangle2DList(pdDocument, - pageNumber, - cbiAddressRects, - PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.green).build()); + pageNumber, + cbiAddressRects, + PdfVisualisationUtility.Options.builder().stroke(true).strokeColor(Color.green).build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java index 8b10d245..2a66bcbb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java @@ -30,7 +30,13 @@ public class RegExPatternTest { @Test public void testEmailRegEx() { - String text = "Address: Schwarzwaldalle " + "P.O.Box\n" + "CH-4002 Basel\n" + "Switzerland\n" + "Contact: Christian Warmers\n" + "Tel: +41 (61) 323 8044\n" + "christian.warmers@syngenta.com"; + String text = "Address: Schwarzwaldalle " + + "P.O.Box\n" + + "CH-4002 Basel\n" + + "Switzerland\n" + + "Contact: Christian Warmers\n" + + "Tel: +41 (61) 323 8044\n" + + "christian.warmers@syngenta.com"; Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern.CASE_INSENSITIVE); @@ -46,7 +52,31 @@ public class RegExPatternTest { @Test public void testEtAlRegEx() { - String text = "To assess the potential of S-metolachlor to cause endocrine disruption (ED) a review (Charlton 2014,\n" + "ASB2016-762) was submitted that summarises results from regulatory and open scientific literature\n" + "studies covering in vitro and in vivo studies (level 2-5 of the OECD Conceptual Framework). According to this information metolachlor increased (1.5-fold) aromatase activity in JEG-3 cells (Laville et al.\n" + "2006, ASB2010-14391) and induced weak anti-androgenic activity in the MDA-kb2 reporter cell line\n" + "with a IC50 of 9.92 µM (IC50 of positive control flutamide: 0.51 µM) (Aït-Aïssa et al. 2010, ASB2015-\n" + "9562). Data from the Tox21 high throughput screening revealed just few postive findings in assays to\n" + "identify antagonists of the androgen receptor. An isolated result of this screening showed agonistic\n" + "activity on the thyroid stimulating hormone receptor, while Dalton et al. (2003, ASB2018-2832)\n" + "demonstrated that metolachlor induced CYP2B1/2 and CYP3A1/2 but did not affect T4, T3 or TSH.\n" + "After prepubertal exposure of male Wistar rats to metolachlor (Mathias et al. 2012, ASB2016-9890) a\n" + "statistically significant increase of serum hormone concentration was observed for testosterone (at the\n" + "dose 50 mg/kg) as well as a statistically significant decrease in the age of preputial separation at a dose\n" + "of 5 and 50 mg/kg. Furthermore a statistically significant increase for estradiol at a dose of 50 mg/kg\n" + "and for FSH at a dose of 5 and 50 mg/kg and morphological alterations of the seminiferous epithelium\n" + "were observed. Relative testicular weight was not altered. A statistically significant increase of relative\n" + "weights was observed in long-term studies with rats (Tisdel et al. 1983, TOX9800328 ). This finding\n" + "was attributed to lower terminal body weight. In mice a statistically significant decrease of the weight\n" + "seminal vesicle (Tisdel et al. 1982, TOX9800327) was shown after 24 month treatment with\n" + "metolachlor. In a mouse preimplantation embryo assay from open literature metolachlor increased the\n" + "percentage of apoptosis significantly and reduced the mean number of cells per embryo significantly\n" + "while the percentage of developing blastocytes was unaltered (Grennlee et al. 2004, ASB2016-9889).\n" + "In reproduvtive toxicity studies a retarded body weight development of the pups was observed, while\n" + "survival and normal morphological and functional development were not altered. No adverse effects\n" + "on male fertility were seen, however important parameters to assess effects on female fertility like\n" + "cyclicity, ovarian follicles as well as developmental landmarks in the offspring have not been investigated."; + String text = "To assess the potential of S-metolachlor to cause endocrine disruption (ED) a review (Charlton 2014,\n" + + "ASB2016-762) was submitted that summarises results from regulatory and open scientific literature\n" + + "studies covering in vitro and in vivo studies (level 2-5 of the OECD Conceptual Framework). According to this information metolachlor increased (1.5-fold) aromatase activity in JEG-3 cells (Laville et al.\n" + + "2006, ASB2010-14391) and induced weak anti-androgenic activity in the MDA-kb2 reporter cell line\n" + + "with a IC50 of 9.92 µM (IC50 of positive control flutamide: 0.51 µM) (Aït-Aïssa et al. 2010, ASB2015-\n" + + "9562). Data from the Tox21 high throughput screening revealed just few postive findings in assays to\n" + + "identify antagonists of the androgen receptor. An isolated result of this screening showed agonistic\n" + + "activity on the thyroid stimulating hormone receptor, while Dalton et al. (2003, ASB2018-2832)\n" + + "demonstrated that metolachlor induced CYP2B1/2 and CYP3A1/2 but did not affect T4, T3 or TSH.\n" + + "After prepubertal exposure of male Wistar rats to metolachlor (Mathias et al. 2012, ASB2016-9890) a\n" + + "statistically significant increase of serum hormone concentration was observed for testosterone (at the\n" + + "dose 50 mg/kg) as well as a statistically significant decrease in the age of preputial separation at a dose\n" + + "of 5 and 50 mg/kg. Furthermore a statistically significant increase for estradiol at a dose of 50 mg/kg\n" + + "and for FSH at a dose of 5 and 50 mg/kg and morphological alterations of the seminiferous epithelium\n" + + "were observed. Relative testicular weight was not altered. A statistically significant increase of relative\n" + + "weights was observed in long-term studies with rats (Tisdel et al. 1983, TOX9800328 ). This finding\n" + + "was attributed to lower terminal body weight. In mice a statistically significant decrease of the weight\n" + + "seminal vesicle (Tisdel et al. 1982, TOX9800327) was shown after 24 month treatment with\n" + + "metolachlor. In a mouse preimplantation embryo assay from open literature metolachlor increased the\n" + + "percentage of apoptosis significantly and reduced the mean number of cells per embryo significantly\n" + + "while the percentage of developing blastocytes was unaltered (Grennlee et al. 2004, ASB2016-9889).\n" + + "In reproduvtive toxicity studies a retarded body weight development of the pups was observed, while\n" + + "survival and normal morphological and functional development were not altered. No adverse effects\n" + + "on male fertility were seen, however important parameters to assess effects on female fertility like\n" + + "cyclicity, ovarian follicles as well as developmental landmarks in the offspring have not been investigated."; Pattern p = Pattern.compile("([^\\s(]*?( \\w\\.?)?) et al\\.?"); @@ -64,7 +94,6 @@ public class RegExPatternTest { String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO"; - Pattern pattern = Pattern.compile("[A-ZÄÖÜ][\\wäöüéèê]{2,}( [A-ZÄÖÜ]{1,2}\\.)+"); Matcher matcher = pattern.matcher(word); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java index f8b513a0..1d2c245b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java @@ -27,7 +27,8 @@ public final class ResourceLoader { } catch (Exception e) { throw new IllegalArgumentException("could not load classpath resource: " + name, e); } - return files.stream().collect(Collectors.toMap(ResourceLoader::getFileName, s -> name + s)); + return files.stream() + .collect(Collectors.toMap(ResourceLoader::getFileName, s -> name + s)); } @@ -44,7 +45,8 @@ public final class ResourceLoader { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); } try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - return br.lines().collect(Collectors.toSet()); + return br.lines() + .collect(Collectors.toSet()); } catch (IOException e) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); } @@ -58,7 +60,8 @@ public final class ResourceLoader { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); } try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - return br.lines().collect(Collectors.joining("\n")); + return br.lines() + .collect(Collectors.joining("\n")); } catch (IOException e) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/Cbi11Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/Cbi11Test.java index c0a5b46c..61013efb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/Cbi11Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/Cbi11Test.java @@ -22,11 +22,16 @@ public class Cbi11Test extends RulesIntegrationTest { doAnalysis(document, Collections.emptyList()); - List authorNames = document.getEntities().stream().map(Dictionary::splitIntoAuthorNames).flatMap(Collection::stream).toList(); + List authorNames = document.getEntities() + .stream() + .map(Dictionary::splitIntoAuthorNames) + .flatMap(Collection::stream) + .toList(); assertThat(authorNames).containsExactlyInAnyOrder("Cargile", "N.L.", "Ross", "J.A.", "Egli", "Ramsteiner"); } + @Test public void testAndIsNotAnAuthorName() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java index b6cbf355..5644c98a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java @@ -41,7 +41,8 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { protected void doAnalysis(Document document, Collection objectToInsert) { kieSession.insert(document); - document.streamAllSubNodes().forEach(kieSession::insert); + document.streamAllSubNodes() + .forEach(kieSession::insert); objectToInsert.forEach(kieSession::insert); kieSession.fireAllRules(); kieSession.dispose(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparatorsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparatorsTest.java index 39ae7fcd..060d8feb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparatorsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparatorsTest.java @@ -16,7 +16,10 @@ class EntityComparatorsTest { Entity entity1 = Entity.builder().startOffset(0).build(); Entity entity2 = Entity.builder().startOffset(2).build(); Entity entity3 = Entity.builder().startOffset(20).build(); - assertEquals(entity1, Stream.of(entity2, entity3, entity1).min(EntityComparators.first()).orElseThrow()); + assertEquals(entity1, + Stream.of(entity2, entity3, entity1) + .min(EntityComparators.first()) + .orElseThrow()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java index 1c1a5800..e2cfe0a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java @@ -16,8 +16,8 @@ class SemanticNodeComparatorsTest { @Test public void testFirstSemanticNode() { - var node = new Section(List.of(0, 1), null, null, null, null,null); - var otherNode = new Section(List.of(0, 2), null, null, null, null,null); + var node = new Section(List.of(0, 1), null, null, null, null, null); + var otherNode = new Section(List.of(0, 2), null, null, null, null, null); List list = new ArrayList<>(); list.add(otherNode); list.add(node); @@ -55,8 +55,8 @@ class SemanticNodeComparatorsTest { @Test public void testFirstSemanticNode4() { - var node = new Section(List.of(1, 5, 8), null, null, null, null,null); - var otherNode = new Section(List.of(1, 5, 9), null, null, null, null,null); + var node = new Section(List.of(1, 5, 8), null, null, null, null, null); + var otherNode = new Section(List.of(1, 5, 9), null, null, null, null, null); List list = new ArrayList<>(); list.add(otherNode); list.add(node); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index 8eeba13c..3a87d2ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -84,6 +84,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT @SpyBean RabbitTemplate rabbitTemplate; + @BeforeEach public void stubClients() { @@ -101,21 +102,22 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } + @Test @SneakyThrows public void testManualSurroundingText() { @@ -125,10 +127,20 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()), "AOEL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(355.53775f) + .topLeftY(266.1895f) + .width(29.32224f) + .height(10.048125f) + .page(1) + .build()), + "AOEL"); var notFoundId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), "Random"); + ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(notFoundId, + List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()), + "Random"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); @@ -147,30 +159,43 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertFalse(unprocessedManualEntities.isEmpty()); assertEquals(unprocessedManualEntities.size(), 2); - Optional optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst(); + Optional optionalUnprocessedManualEntity = unprocessedManualEntities.stream() + .filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)) + .findFirst(); assertTrue(optionalUnprocessedManualEntity.isPresent()); UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get(); assertEquals(unprocessedManualEntity.getTextBefore(), "was above the "); assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f); - assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f); - assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f); - assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).x(), 355.53775f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).y(), 266.49002f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).w(), 29.322266f); + assertEquals(unprocessedManualEntity.getPositions() + .get(0).h(), 11.017679f); - Optional optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst(); + Optional optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream() + .filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)) + .findFirst(); assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent()); UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get(); assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), ""); assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), ""); assertEquals(unprocessedNotFoundManualEntity.getSection(), ""); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f); - assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getPageNumber(), 1); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[0], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[1], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[2], 1f); + assertEquals(unprocessedNotFoundManualEntity.getPositions() + .get(0).getRectangle()[3], 1f); analyzeService.reanalyze(request); @@ -190,10 +215,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance "); assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).h(), positions.get(0).getHeight()); } @@ -205,13 +234,37 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); var cormsId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(35.9904f).height(10.048125f).page(1).build()), "CoRMS"); + ManualRedactionEntry manualRedactionEntry2 = prepareManualRedactionEntry(cormsId, + List.of(Rectangle.builder() + .topLeftX(129.86f) + .topLeftY(505.7295f) + .width(35.9904f) + .height(10.048125f) + .page(1) + .build()), + "CoRMS"); var a9Id = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, List.of(Rectangle.builder().topLeftX(140.1096f).topLeftY(291.5095f).width(37.84512f).height(10.048125f).page(1).build()), "A9396G"); + ManualRedactionEntry manualRedactionEntry3 = prepareManualRedactionEntry(a9Id, + List.of(Rectangle.builder() + .topLeftX(140.1096f) + .topLeftY(291.5095f) + .width(37.84512f) + .height(10.048125f) + .page(1) + .build()), + "A9396G"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry3); manualRedactions.getEntriesToAdd().add(manualRedactionEntry2); @@ -238,35 +291,53 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertFalse(unprocessedManualEntities.isEmpty()); assertEquals(unprocessedManualEntities.size(), 3); - var resizedAoel = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)).findFirst(); + var resizedAoel = unprocessedManualEntities.stream() + .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(aoelId)) + .findFirst(); assertTrue(resizedAoel.isPresent()); assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of"); assertEquals(resizedAoel.get().getTextBefore(), "is below the "); assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(resizedAoel.get().getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(resizedAoel.get().getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(resizedAoel.get().getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(resizedAoel.get().getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(resizedAoel.get().getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(resizedAoel.get().getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(resizedAoel.get().getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(resizedAoel.get().getPositions() + .get(0).h(), positions.get(0).getHeight()); - var cormsResized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)).findFirst(); + var cormsResized = unprocessedManualEntities.stream() + .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(cormsId)) + .findFirst(); assertTrue(cormsResized.isPresent()); assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of"); assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore "); assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the"); - assertEquals(cormsResized.get().getPositions().get(0).x(), positions2.get(0).getTopLeftX()); - assertEquals(cormsResized.get().getPositions().get(0).y(), positions2.get(0).getTopLeftY()); - assertEquals(cormsResized.get().getPositions().get(0).w(), positions2.get(0).getWidth()); - assertEquals(cormsResized.get().getPositions().get(0).h(), positions2.get(0).getHeight()); + assertEquals(cormsResized.get().getPositions() + .get(0).x(), positions2.get(0).getTopLeftX()); + assertEquals(cormsResized.get().getPositions() + .get(0).y(), positions2.get(0).getTopLeftY()); + assertEquals(cormsResized.get().getPositions() + .get(0).w(), positions2.get(0).getWidth()); + assertEquals(cormsResized.get().getPositions() + .get(0).h(), positions2.get(0).getHeight()); - var a9Resized = unprocessedManualEntities.stream().filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)).findFirst(); + var a9Resized = unprocessedManualEntities.stream() + .filter(unprocessedManualEntity -> unprocessedManualEntity.getAnnotationId().equals(a9Id)) + .findFirst(); assertTrue(a9Resized.isPresent()); assertEquals(a9Resized.get().getTextAfter(), " were obtained from"); assertEquals(a9Resized.get().getTextBefore(), "data for S"); assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(a9Resized.get().getPositions().get(0).x(), positions3.get(0).getTopLeftX()); - assertEquals(a9Resized.get().getPositions().get(0).y(), positions3.get(0).getTopLeftY()); - assertEquals(a9Resized.get().getPositions().get(0).w(), positions3.get(0).getWidth()); - assertEquals(a9Resized.get().getPositions().get(0).h(), positions3.get(0).getHeight()); + assertEquals(a9Resized.get().getPositions() + .get(0).x(), positions3.get(0).getTopLeftX()); + assertEquals(a9Resized.get().getPositions() + .get(0).y(), positions3.get(0).getTopLeftY()); + assertEquals(a9Resized.get().getPositions() + .get(0).w(), positions3.get(0).getWidth()); + assertEquals(a9Resized.get().getPositions() + .get(0).h(), positions3.get(0).getHeight()); } @@ -277,7 +348,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); AnalyzeRequest request = uploadFileToStorage(pdfFile); @@ -301,10 +380,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of"); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the "); assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).h(), positions.get(0).getHeight()); } @@ -315,7 +398,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); AnalyzeRequest request = uploadFileToStorage(pdfFile); @@ -339,10 +430,14 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same"); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an "); assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the"); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).x(), positions.get(0).getTopLeftX()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).y(), positions.get(0).getTopLeftY()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).w(), positions.get(0).getWidth()); - assertEquals(unprocessedManualEntities.get(0).getPositions().get(0).h(), positions.get(0).getHeight()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).x(), positions.get(0).getTopLeftX()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).y(), positions.get(0).getTopLeftY()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).w(), positions.get(0).getWidth()); + assertEquals(unprocessedManualEntities.get(0).getPositions() + .get(0).h(), positions.get(0).getHeight()); } @@ -353,7 +448,15 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT ManualRedactions manualRedactions = new ManualRedactions(); var aoelId = UUID.randomUUID().toString(); - ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, List.of(Rectangle.builder().topLeftX(384.85536f).topLeftY(240.8695f).width(13.49088f).height(10.048125f).page(1).build()), "EL"); + ManualRedactionEntry manualRedactionEntry = prepareManualRedactionEntry(aoelId, + List.of(Rectangle.builder() + .topLeftX(384.85536f) + .topLeftY(240.8695f) + .width(13.49088f) + .height(10.048125f) + .page(1) + .build()), + "EL"); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); AnalyzeRequest request = uploadFileToStorage(pdfFile); @@ -398,4 +501,5 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT manualRedactionEntry.setPositions(positions); return manualRedactionEntry; } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java index 8b7983d7..4b71dac7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java @@ -35,7 +35,9 @@ public class StringMatchingPerformanceTest { System.out.println("Loaded text has a length of " + text.length() + " symbols"); System.out.println("Dictionary has " + dictionary.size() + " entries"); - var patterns = dictionary.stream().map(p -> Pattern.compile(Pattern.quote(p))).toList(); + var patterns = dictionary.stream() + .map(p -> Pattern.compile(Pattern.quote(p))) + .toList(); var trie = Trie.builder().ignoreCase().addKeywords(dictionary).build(); // 1. Naive approach @@ -57,7 +59,9 @@ public class StringMatchingPerformanceTest { t1 = System.currentTimeMillis(); var boyerMooreIndexes = new HashSet(); for (var pattern : patterns) { - boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).toList()); + boyerMooreIndexes.addAll(pattern.matcher(text).results() + .map(r -> new Index(r.start(), r.end())) + .toList()); } t2 = System.currentTimeMillis(); System.out.println("Boyer Moore found " + boyerMooreIndexes.size() + " entries in " + (t2 - t1) + "ms"); @@ -65,7 +69,9 @@ public class StringMatchingPerformanceTest { // 3. Aho Corasick t1 = System.currentTimeMillis(); var result = trie.parseText(text); - var ahoCorasickIndexes = result.stream().map(r -> new Index(r.getStart(), r.getEnd() + 1)).collect(Collectors.toSet()); + var ahoCorasickIndexes = result.stream() + .map(r -> new Index(r.getStart(), r.getEnd() + 1)) + .collect(Collectors.toSet()); t2 = System.currentTimeMillis(); System.out.println("Aho Corasick found " + ahoCorasickIndexes.size() + " entries in " + (t2 - t1) + "ms"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/MetricValidationUtils.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/MetricValidationUtils.java index d36714b2..838b8e6e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/MetricValidationUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/MetricValidationUtils.java @@ -13,7 +13,10 @@ public class MetricValidationUtils { public static void validateMetric(PrometheusMeterRegistry registry, String name, int invocationCount, Integer time) { - var metricOptional = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase(name)).findAny(); + var metricOptional = registry.getMeters() + .stream() + .filter(m -> m.getId().getName().equalsIgnoreCase(name)) + .findAny(); assertThat(metricOptional.isPresent()).isTrue(); PrometheusTimer metric = (PrometheusTimer) metricOptional.get(); assertThat(metric.count()).isGreaterThanOrEqualTo(invocationCount); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java index 85771fa3..006e1e20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java @@ -34,7 +34,8 @@ public class PdfVisualisationUtility { public void drawDocumentGraph(PDDocument document, Document documentGraph) { - documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry)); + documentGraph.getDocumentTree().allEntriesInOrder() + .forEach(entry -> drawNode(document, entry)); } @@ -49,13 +50,19 @@ public class PdfVisualisationUtility { public void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) { - textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options)); + textBlock.getAtomicTextBlocks() + .forEach(atb -> drawAtomicTextBlock(document, atb, options)); } public void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) { - drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options); + drawRectangle2DList(document, + atomicTextBlock.getPage().getNumber(), + atomicTextBlock.getPositions() + .stream() + .toList(), + options); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java index d26d0bf5..7267bd20 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java @@ -105,19 +105,20 @@ public class Main { case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType); default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t"))); }; - identifiersPerFile.keySet().forEach(path -> { - String ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiersPerFile.get(path), applicationType); + identifiersPerFile.keySet() + .forEach(path -> { + String ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiersPerFile.get(path), applicationType); - ruleFileString = escapeRuleStringIfOptionIsSet(cmd, ruleFileString); - Path output = outputDir.resolve(path); - output.getParent().toFile().mkdirs(); - try (var out = new FileOutputStream(output.toFile())) { - out.write(ruleFileString.getBytes(StandardCharsets.UTF_8)); - } catch (IOException e) { - System.out.println(e.getMessage()); - System.exit(1); - } - }); + ruleFileString = escapeRuleStringIfOptionIsSet(cmd, ruleFileString); + Path output = outputDir.resolve(path); + output.getParent().toFile().mkdirs(); + try (var out = new FileOutputStream(output.toFile())) { + out.write(ruleFileString.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + System.out.println(e.getMessage()); + System.exit(1); + } + }); } @@ -172,18 +173,26 @@ public class Main { Options options = new Options(); Option type = new Option("t",// - "type",// - true,// - "Please specify the input type, defaults to \"list\", \n" +// - "current options:\n" +// - "\"list\" or \"l\" (List of Rule identifiers e.g. \"MAN, X, LDS, CBI.0, PII.0.1\")\n" +// - "\"file\" or \"f\" (Absolute Path to rule file, escaped or not, but the rule file must contain the current Rule Identifiers, the main use case is migrating rule files to the current state)\n" +// - "\"old\" or \"o\" (Absolute Path to old rule file, escaped or not, the script will attempt to translate the old rules to new rules based on \"main/resources/old_rules_with_translations.csv\")\n"); + "type",// + true,// + "Please specify the input type, defaults to \"list\", \n" + + +// + "current options:\n" + + +// + "\"list\" or \"l\" (List of Rule identifiers e.g. \"MAN, X, LDS, CBI.0, PII.0.1\")\n" + + +// + "\"file\" or \"f\" (Absolute Path to rule file, escaped or not, but the rule file must contain the current Rule Identifiers, the main use case is migrating rule files to the current state)\n" + + +// + "\"old\" or \"o\" (Absolute Path to old rule file, escaped or not, the script will attempt to translate the old rules to new rules based on \"main/resources/old_rules_with_translations.csv\")\n"); options.addOption(type); Option input = new Option("i", - "input", - true, - "Based on type expects: String of Rule identifiers, Absolute Path to a rules file or a folder containing rules files named rules.txt or rules.drl if \"recursive\" flag is set"); + "input", + true, + "Based on type expects: String of Rule identifiers, Absolute Path to a rules file or a folder containing rules files named rules.txt or rules.drl if \"recursive\" flag is set"); input.setRequired(true); options.addOption(input); @@ -192,23 +201,23 @@ public class Main { options.addOption(output); Option noTranslate = new Option("n", - "noop", - false, - "If this flag is set, the input file will not be attempted to be translated or migrated, use this to only escape files. Currently this only works with input type \"file\""); + "noop", + false, + "If this flag is set, the input file will not be attempted to be translated or migrated, use this to only escape files. Currently this only works with input type \"file\""); options.addOption(noTranslate); Option escapeOutput = new Option("e", "escape", false, "If this flag is set, the output files will be escaped and saved as \".txt\", like in the dossier templates"); options.addOption(escapeOutput); Option recursive = new Option("r", - "recursive", - false, - "Can only be set in combination with type \"file\" or \"old\" Searches recursively for files named rules.txt or rules.drl and attempts to migrate/translate them."); + "recursive", + false, + "Can only be set in combination with type \"file\" or \"old\" Searches recursively for files named rules.txt or rules.drl and attempts to migrate/translate them."); options.addOption(recursive); Option application = new Option("a", - "application", - true, - "What application to update the rules for. For Redact Manager specify: RM. For Documine specify: DM. Defaults to RM."); + "application", + true, + "What application to update the rules for. For Redact Manager specify: RM. For Documine specify: DM. Defaults to RM."); options.addOption(application); return options; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java index 8f255e8e..af844a6d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java @@ -17,7 +17,7 @@ public class RuleManagementResources { if (applicationType == ApplicationType.RM) { return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl"); } - return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); + return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl"); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java index 0019fd1f..2a8de641 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java @@ -19,7 +19,9 @@ public class DefaultRuleIdentifiersFactory { try (var defaultRuleIdentifiers = RuleManagementResources.getDefaultRuleIdentifiesInputStream(applicationType)) { String ruleIdentifierString = new String(defaultRuleIdentifiers.readAllBytes()); - return Arrays.stream(ruleIdentifierString.split("\n")).map(RuleIdentifier::fromString).collect(Collectors.toSet()); + return Arrays.stream(ruleIdentifierString.split("\n")) + .map(RuleIdentifier::fromString) + .collect(Collectors.toSet()); } } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java index 17933191..039664bf 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java @@ -28,7 +28,10 @@ public class RuleFileFactory { if (identifiers.isBlank() || identifiers.isEmpty()) { return createFileFromIdentifiers(emptySet(), applicationType); } - return createFileFromIdentifiers(Arrays.stream(identifiers.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet()), applicationType); + return createFileFromIdentifiers(Arrays.stream(identifiers.split(",")) + .map(String::trim) + .map(RuleIdentifier::fromString) + .collect(Collectors.toSet()), applicationType); } @@ -63,13 +66,19 @@ public class RuleFileFactory { private List parseRuleOrder(String template) { String[] values = template.split("\n"); - return Arrays.stream(values).map(RuleType::fromString).toList(); + return Arrays.stream(values) + .map(RuleType::fromString) + .toList(); } private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List ruleOrder) { - Set additionalRuleTypes = bluePrint.ruleClasses().stream().map(RuleClass::ruleType).filter(ruleType -> !ruleOrder.contains(ruleType)).collect(Collectors.toSet()); + Set additionalRuleTypes = bluePrint.ruleClasses() + .stream() + .map(RuleClass::ruleType) + .filter(ruleType -> !ruleOrder.contains(ruleType)) + .collect(Collectors.toSet()); StringBuilder sb = new StringBuilder(); sb.append(bluePrint.imports()); @@ -82,7 +91,9 @@ public class RuleFileFactory { sb.append("\n\n"); for (RuleType ruleBlockType : ruleOrder) { if (ruleBlockType.isWildCard()) { - additionalRuleTypes.stream().sorted(Comparator.comparing(RuleType::name)).forEach(ruleType -> writeRuleClass(bluePrint, ruleType, sb)); + additionalRuleTypes.stream() + .sorted(Comparator.comparing(RuleType::name)) + .forEach(ruleType -> writeRuleClass(bluePrint, ruleType, sb)); continue; } if (bluePrint.findRuleClassByType(ruleBlockType).isEmpty()) { @@ -100,19 +111,27 @@ public class RuleFileFactory { sb.append(ruleType); sb.append(" rules ------------------------------------"); sb.append("\n\n"); - RuleClass ruleClass = bluePrint.findRuleClassByType(ruleType).orElseThrow(); - List sortedRuleUnits = ruleClass.ruleUnits().stream().sorted(Comparator.comparingInt(RuleUnit::unit)).toList(); + RuleClass ruleClass = bluePrint.findRuleClassByType(ruleType) + .orElseThrow(); + List sortedRuleUnits = ruleClass.ruleUnits() + .stream() + .sorted(Comparator.comparingInt(RuleUnit::unit)) + .toList(); for (RuleUnit unit : sortedRuleUnits) { if (unit.rules().isEmpty()) { continue; } sb.append("// "); - sb.append(unit.rules().get(0).identifier().toRuleUnitString()); + sb.append(unit.rules() + .get(0).identifier().toRuleUnitString()); sb.append("\n"); - unit.rules().stream().sorted(Comparator.comparingInt(rule -> rule.identifier().id())).forEach(rule -> { - sb.append(rule.code()); - sb.append("\n\n"); - }); + unit.rules() + .stream() + .sorted(Comparator.comparingInt(rule -> rule.identifier().id())) + .forEach(rule -> { + sb.append(rule.code()); + sb.append("\n\n"); + }); sb.append("\n"); } } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java index feb60f62..3e17b0db 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java @@ -57,7 +57,12 @@ public class RuleFileParser { } allRules.add(BasicRule.fromRuleDescr(rule, rulesString)); } - String imports = rulesString.substring(0, packageDescr.getImports().stream().mapToInt(ImportDescr::getEndCharacter).max().orElseThrow() + 1); + String imports = rulesString.substring(0, + packageDescr.getImports() + .stream() + .mapToInt(ImportDescr::getEndCharacter) + .max() + .orElseThrow() + 1); String globals = packageDescr.getGlobals() .stream() .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) @@ -71,15 +76,24 @@ public class RuleFileParser { private List buildRuleClasses(List allRules) { - Map> rulesPerType = allRules.stream().collect(groupingBy(rule -> rule.identifier().type())); - return rulesPerType.keySet().stream().map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))).collect(Collectors.toList()); + Map> rulesPerType = allRules.stream() + .collect(groupingBy(rule -> rule.identifier().type())); + return rulesPerType.keySet() + .stream() + .map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))) + .collect(Collectors.toList()); } private List groupingByGroup(List rules) { - Map> rulesPerUnit = rules.stream().collect(groupingBy(rule -> rule.identifier().unit())); - return rulesPerUnit.keySet().stream().sorted().map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))).collect(Collectors.toList()); + Map> rulesPerUnit = rules.stream() + .collect(groupingBy(rule -> rule.identifier().unit())); + return rulesPerUnit.keySet() + .stream() + .sorted() + .map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))) + .collect(Collectors.toList()); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java index aac63fa5..fc8e9026 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java @@ -80,9 +80,12 @@ public class RuleIdentifierMigrator { String migratedCode = basicRule.code() .replaceAll("\\.addMatchedRule\\(.*\\)", - Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis))); + Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis))); migratedCode = migratedCode.replaceAll("\\.setMatchedRule\\(.*\\)", - Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis))); + Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", + basicRule.identifier().toString(), + redactionReason, + legalBasis))); migratedCode = migratedCode.replaceAll("\\.setRedactionReason\\(\".*\"\\)", ""); migratedCode = migratedCode.replaceAll("\\.setLegalBasis\\(\".*\"\\)", ""); migratedCode = migratedCode.replaceAll("\\$entity;\n", ""); @@ -117,9 +120,9 @@ public class RuleIdentifierMigrator { public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List records) { - BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()).orElseThrow() - .findRuleUnitByInteger(oldIdentifier.unit()).orElseThrow() - .rules() + BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()) + .orElseThrow().findRuleUnitByInteger(oldIdentifier.unit()) + .orElseThrow().rules() .stream() .filter(rule -> rule.identifier().equals(oldIdentifier)) .findFirst() diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java index 5383b80e..8fdac0d7 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java @@ -10,6 +10,7 @@ public enum ApplicationType { @Getter private final String value; + ApplicationType(String value) { this.value = value; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java index 1ce158ed..9d77bbb6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java @@ -5,6 +5,7 @@ import org.drools.drl.ast.descr.RuleDescr; import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO; public record BasicRule(RuleIdentifier identifier, String name, String code) { + public static BasicRule fromRuleDescr(RuleDescr rule, String rulesString) { RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName()); @@ -13,6 +14,7 @@ public record BasicRule(RuleIdentifier identifier, String name, String code) { return new BasicRule(identifier, nameWithoutIdentifier, code); } + public String toEscapedCode() { return RuleFileIO.escapeAndWrap(code()); diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java index ede53a1a..40468cc0 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java @@ -9,7 +9,9 @@ public record RuleClass(RuleType ruleType, List ruleUnits) { public Optional findRuleUnitByInteger(Integer unit) { - return ruleUnits.stream().filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst(); + return ruleUnits.stream() + .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)) + .findFirst(); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java index 045cfb22..5b1abd6d 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java @@ -13,28 +13,36 @@ public record RuleFileBluePrint(String imports, String globals, String queries, public void removeRule(RuleIdentifier ruleIdentifier) { - findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()).ifPresent(ruleUnit -> { - ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); - if (ruleUnit.rules().isEmpty()) { - ruleClass.ruleUnits().remove(ruleUnit); - } - if (ruleClass.ruleUnits().isEmpty()) { - ruleClasses().remove(ruleClass); - } - })); + findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()) + .ifPresent(ruleUnit -> { + ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier)); + if (ruleUnit.rules().isEmpty()) { + ruleClass.ruleUnits().remove(ruleUnit); + } + if (ruleClass.ruleUnits().isEmpty()) { + ruleClasses().remove(ruleClass); + } + })); } public Optional findRuleClassByType(RuleType ruleType) { - return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst(); + return ruleClasses.stream() + .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)) + .findFirst(); } public Set getAllRules() { - return ruleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream).collect(Collectors.toSet()); + return ruleClasses().stream() + .map(RuleClass::ruleUnits) + .flatMap(Collection::stream) + .map(RuleUnit::rules) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); } @@ -48,7 +56,9 @@ public record RuleFileBluePrint(String imports, String globals, String queries, Set newRuleIdentifiers = ruleFileBluePrint.getAllRuleIdentifiers(); Set existingRuleIdentifiers = this.getAllRuleIdentifiers(); - Set duplicatedRuleIdentifiers = existingRuleIdentifiers.stream().filter(newRuleIdentifiers::contains).collect(Collectors.toSet()); + Set duplicatedRuleIdentifiers = existingRuleIdentifiers.stream() + .filter(newRuleIdentifiers::contains) + .collect(Collectors.toSet()); for (RuleIdentifier duplicatedRuleIdentifier : duplicatedRuleIdentifiers) { List thisRules = findRuleByIdentifier(duplicatedRuleIdentifier); List otherRules = ruleFileBluePrint.findRuleByIdentifier(duplicatedRuleIdentifier); @@ -59,19 +69,27 @@ public record RuleFileBluePrint(String imports, String globals, String queries, } } newRuleIdentifiers.removeAll(existingRuleIdentifiers); - newRuleIdentifiers.forEach(identifierToAdd -> ruleFileBluePrint.findRuleByIdentifier(identifierToAdd).forEach(this::addRule)); + newRuleIdentifiers.forEach(identifierToAdd -> ruleFileBluePrint.findRuleByIdentifier(identifierToAdd) + .forEach(this::addRule)); } public List findRuleByIdentifier(RuleIdentifier ruleIdentifier) { if (Objects.isNull(ruleIdentifier.unit())) { - return findRuleClassByType(ruleIdentifier.type()).map(ruleClass -> ruleClass.ruleUnits().stream().flatMap(ruleUnit -> ruleUnit.rules().stream()).toList()) + return findRuleClassByType(ruleIdentifier.type()).map(ruleClass -> ruleClass.ruleUnits() + .stream() + .flatMap(ruleUnit -> ruleUnit.rules() + .stream()) + .toList()) .orElse(Collections.emptyList()); } return findRuleClassByType(ruleIdentifier.type())// .map(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit())// - .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.identifier().matches(ruleIdentifier)).toList())// + .map(ruleUnit -> ruleUnit.rules() + .stream() + .filter(rule -> rule.identifier().matches(ruleIdentifier)) + .toList())// .orElse(Collections.emptyList()))// .orElse(Collections.emptyList()); } @@ -93,7 +111,8 @@ public record RuleFileBluePrint(String imports, String globals, String queries, RuleClass ruleClass = findRuleClassByType(rule.identifier().type()).orElseGet(() -> createNewRuleClass(rule)); - RuleUnit ruleUnit = ruleClass.findRuleUnitByInteger(rule.identifier().unit()).orElseGet(() -> ruleClass.createNewRuleUnit(rule.identifier().unit())); + RuleUnit ruleUnit = ruleClass.findRuleUnitByInteger(rule.identifier().unit()) + .orElseGet(() -> ruleClass.createNewRuleUnit(rule.identifier().unit())); ruleUnit.rules().add(rule); } @@ -103,9 +122,12 @@ public record RuleFileBluePrint(String imports, String globals, String queries, RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>()); ruleClasses().stream() - .flatMap(ruleClass -> ruleClass.ruleUnits().stream()) - .flatMap(ruleUnit -> ruleUnit.rules().stream()) - .filter(rule -> identifiers.stream().anyMatch(identifier -> rule.identifier().matches(identifier))) + .flatMap(ruleClass -> ruleClass.ruleUnits() + .stream()) + .flatMap(ruleUnit -> ruleUnit.rules() + .stream()) + .filter(rule -> identifiers.stream() + .anyMatch(identifier -> rule.identifier().matches(identifier))) .forEach(filteredBluePrint::addRule); return filteredBluePrint; } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java index a747f90d..1c863af2 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java @@ -38,7 +38,10 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { public static Set fromListOfIdentifiersString(String input) { - return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet()); + return Arrays.stream(input.split(",")) + .map(String::trim) + .map(RuleIdentifier::fromString) + .collect(Collectors.toSet()); } @@ -60,8 +63,8 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { public boolean matches(RuleIdentifier ruleIdentifier) { return ruleIdentifier.type().equals(this.type()) && // - (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // - (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); + (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // + (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java index 46ff26cf..d14aa1a0 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java @@ -14,14 +14,14 @@ import lombok.experimental.FieldDefaults; public class RuleType { static Map fullNameMap = Map.of("SYN", "Syngenta specific",// - "ETC", "Other",// - "MAN", "Manual changes",// - "X", "Entity merging",// - "FA", "File attributes",// - "LDS", "Local dictionary search",// - "TAB", "Table extraction",// - "H", "Headlines",// - "DOC", "General documine"); + "ETC", "Other",// + "MAN", "Manual changes",// + "X", "Entity merging",// + "FA", "File attributes",// + "LDS", "Local dictionary search",// + "TAB", "Table extraction",// + "H", "Headlines",// + "DOC", "General documine"); @EqualsAndHashCode.Include String name; diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java index bb9a2461..ae0f3134 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java @@ -101,12 +101,20 @@ public class OldRulesParser { RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType); translationPairs.forEach((key, value) -> System.out.printf("Rule %s will be translated to %s \n", - key.name(), - String.format("%s: %s", value, value.stream().map(bluePrint::findRuleByIdentifier).flatMap(Collection::stream).map(BasicRule::name).toList()))); + key.name(), + String.format("%s: %s", + value, + value.stream() + .map(bluePrint::findRuleByIdentifier) + .flatMap(Collection::stream) + .map(BasicRule::name) + .toList()))); return Stream.concat(// - Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// - translationPairs.values().stream().flatMap(Collection::stream)) + Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),// + translationPairs.values() + .stream() + .flatMap(Collection::stream)) .map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null)) .collect(Collectors.toSet()); } @@ -151,9 +159,9 @@ public class OldRulesParser { for (CSVRecord record : records) { String[] values = record.values(); parsedRecords.add(new OldRulesCsvRecord(Long.parseLong(values[0]), - RuleFileIO.unescapeAndUnWrap(values[2]), - RuleFileIO.unescapeAndUnWrap(values[1]), - parseRuleIdentifiers(values[3]))); + RuleFileIO.unescapeAndUnWrap(values[2]), + RuleFileIO.unescapeAndUnWrap(values[1]), + parseRuleIdentifiers(values[3]))); } } return parsedRecords; @@ -182,7 +190,9 @@ public class OldRulesParser { private List parseBooleanList(String[] values) { - return Arrays.stream(values).map(Boolean::parseBoolean).toList(); + return Arrays.stream(values) + .map(Boolean::parseBoolean) + .toList(); } diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java index 8cf1e762..3d3ccac6 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/utils/RuleFileIO.java @@ -65,10 +65,13 @@ public final class RuleFileIO { return rulesString; } - @SneakyThrows - public Stream streamAllRuleFilesInDirectory(Path input){ - return Files.walk(input).filter(path -> path.getFileName().toString().equals("rules.txt") || path.getFileName().toString().equals("rules.drl")).map(Path::toFile); + @SneakyThrows + public Stream streamAllRuleFilesInDirectory(Path input) { + + return Files.walk(input) + .filter(path -> path.getFileName().toString().equals("rules.txt") || path.getFileName().toString().equals("rules.drl")) + .map(Path::toFile); } } diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java index 9385b0ae..f8d25b6a 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/DroolsCompilationTest.java @@ -37,8 +37,8 @@ public class DroolsCompilationTest { assertTrue(validateRuleSyntax(dmRule).isValid()); // Invalidate rules - rmRule = rmRule.substring(0, rmRule.length()-3); - dmRule = dmRule.substring(0, dmRule.length()-3); + rmRule = rmRule.substring(0, rmRule.length() - 3); + dmRule = dmRule.substring(0, dmRule.length() - 3); var validateRM = validateRuleSyntax(rmRule); var validateDM = validateRuleSyntax(dmRule); @@ -48,6 +48,7 @@ public class DroolsCompilationTest { assertFalse(validateDM.getFailReason().contains("mismatched input ''")); } + private RuleValidation validateRuleSyntax(String ruleContent) { RuleValidation.RuleValidationBuilder ruleValidationBuilder = RuleValidation.builder(); @@ -70,11 +71,14 @@ public class DroolsCompilationTest { return ruleValidationBuilder.isValid(true).build(); } + @Data @Builder public static class RuleValidation { private boolean isValid; private String failReason; + } + } diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java index 41e292f1..938d0b5b 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java @@ -36,7 +36,11 @@ public class RuleFileMigrationTest { void migrateAllEntityRules() { for (String ruleFileDir : ruleFileDirs) { - Files.walk(Path.of(ruleFileDir)).filter(this::isEntityRuleFile).map(Path::toFile).peek(System.out::println).forEach(RuleFileMigrator::migrateFile); + Files.walk(Path.of(ruleFileDir)) + .filter(this::isEntityRuleFile) + .map(Path::toFile) + .peek(System.out::println) + .forEach(RuleFileMigrator::migrateFile); } } diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java index 2c835963..d0354ab4 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java @@ -52,15 +52,18 @@ class RuleFileFactoryTest { @SneakyThrows private void migrate(ApplicationType applicationType, String... paths) { - Arrays.stream(paths).forEach(path -> { - try { - Stream.of(Files.walk(Path.of(path))).flatMap(Function.identity()).filter(p -> p.getFileName().toString().equals("rules.drl"))// - .map(Path::toFile)// - .forEach(e -> migrateFile(e, applicationType)); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + Arrays.stream(paths) + .forEach(path -> { + try { + Stream.of(Files.walk(Path.of(path))) + .flatMap(Function.identity()) + .filter(p -> p.getFileName().toString().equals("rules.drl"))// + .map(Path::toFile)// + .forEach(e -> migrateFile(e, applicationType)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } @@ -72,7 +75,8 @@ class RuleFileFactoryTest { String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType); try (FileOutputStream out = new FileOutputStream(oldRulesFile); OutputStreamWriter outWrite = new OutputStreamWriter(out, - StandardCharsets.UTF_8); BufferedWriter writer = new BufferedWriter(outWrite)) { + StandardCharsets.UTF_8); BufferedWriter writer = new BufferedWriter( + outWrite)) { writer.write(newRulesString); } } diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java index c63ff51c..b78ab534 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java @@ -62,10 +62,12 @@ class OldRulesParserTest { public void printTranslationsTest() { List records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream()); - List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")).forEach(type -> { - List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type).orElseThrow().ruleUnits(); - rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); - }); + List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")) + .forEach(type -> { + List rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type) + .orElseThrow().ruleUnits(); + rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records)); + }); } @@ -75,15 +77,21 @@ class OldRulesParserTest { System.out.println("Rule unit empty, skipping!\n"); return; } - RuleIdentifier identifier = ruleUnit.rules().get(0).identifier(); + RuleIdentifier identifier = ruleUnit.rules() + .get(0).identifier(); RuleIdentifier unitIdentifier = new RuleIdentifier(identifier.type(), identifier.unit(), null); String oldNames = records.stream() - .filter(r -> r.translatesTo().stream().anyMatch(i -> i.matches(unitIdentifier))) + .filter(r -> r.translatesTo() + .stream() + .anyMatch(i -> i.matches(unitIdentifier))) .map(OldRulesParser.OldRulesCsvRecord::names) .map(OldRulesParserTest::removeIdFromName) .distinct() .collect(Collectors.joining(", ")); - System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules().stream().map(BasicRule::name).collect(Collectors.joining(", "))); + System.out.println(unitIdentifier.toRuleUnitString() + " " + ruleUnit.rules() + .stream() + .map(BasicRule::name) + .collect(Collectors.joining(", "))); System.out.println("translate from"); System.out.println(oldNames); System.out.println(); @@ -139,11 +147,13 @@ class OldRulesParserTest { // Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa"))) String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/"; Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), - Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), - Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) - // - ).flatMap(Function.identity())// + Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")), + Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")), + Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), + Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) + // + ) + .flatMap(Function.identity())// .filter(path -> path.getFileName().toString().equals("rules.drl"))// .map(Path::toFile)// .forEach(this::translateOldRuleFile); -- 2.47.2