diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index 92f4c3b29..bc7a0d6a0 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -7,8 +7,8 @@ plugins { } val redactionServiceVersion by rootProject.extra { "4.199.0" } -val pdftronRedactionServiceVersion by rootProject.extra { "4.38.0" } -val redactionReportServiceVersion by rootProject.extra { "4.36.0" } +val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" } +val redactionReportServiceVersion by rootProject.extra { "4.47.0" } val searchServiceVersion by rootProject.extra { "2.71.0" } repositories { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/ACLBeanConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/ACLBeanConfiguration.java index 93c32eff3..ee8738a1b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/ACLBeanConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/ACLBeanConfiguration.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.acl; +import com.knecon.fforesight.databasetenantcommons.providers.MultiTenantDataSource; +import lombok.RequiredArgsConstructor; import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -7,7 +9,6 @@ import org.springframework.context.annotation.Primary; import org.springframework.security.access.expression.method.DefaultMethodSecurityExpressionHandler; import org.springframework.security.access.expression.method.MethodSecurityExpressionHandler; import org.springframework.security.acls.AclPermissionCacheOptimizer; -import org.springframework.security.acls.AclPermissionEvaluator; import org.springframework.security.acls.domain.AclAuthorizationStrategy; import org.springframework.security.acls.domain.ConsoleAuditLogger; import org.springframework.security.acls.domain.DefaultPermissionFactory; @@ -20,10 +21,6 @@ import org.springframework.security.acls.jdbc.LookupStrategy; import org.springframework.security.acls.model.PermissionGrantingStrategy; import org.springframework.transaction.annotation.EnableTransactionManagement; -import com.knecon.fforesight.databasetenantcommons.providers.MultiTenantDataSource; - -import lombok.RequiredArgsConstructor; - @Configuration @RequiredArgsConstructor @EnableTransactionManagement @@ -81,7 +78,7 @@ public class ACLBeanConfiguration { public MethodSecurityExpressionHandler defaultMethodSecurityExpressionHandler() { DefaultMethodSecurityExpressionHandler expressionHandler = new DefaultMethodSecurityExpressionHandler(); - AclPermissionEvaluator permissionEvaluator = new AclPermissionEvaluator(aclService()); + CustomAclPermissionEvaluator permissionEvaluator = new CustomAclPermissionEvaluator(aclService()); permissionEvaluator.setPermissionFactory(permissionFactory()); expressionHandler.setPermissionEvaluator(permissionEvaluator); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/CustomAclPermissionEvaluator.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/CustomAclPermissionEvaluator.java index 59f847b33..127555f38 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/CustomAclPermissionEvaluator.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/CustomAclPermissionEvaluator.java @@ -1,18 +1,16 @@ package com.iqser.red.service.persistence.management.v1.processor.acl; -import java.io.Serializable; - +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import org.springframework.security.acls.AclPermissionEvaluator; import org.springframework.security.acls.model.AclService; -import org.springframework.security.acls.model.NotFoundException; import org.springframework.security.core.Authentication; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import java.io.Serializable; public class CustomAclPermissionEvaluator extends AclPermissionEvaluator { public CustomAclPermissionEvaluator(AclService aclService) { - super(aclService); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java index 3ce4e8644..976bb089d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -61,36 +61,36 @@ public class EntityLogMergeService { private final FileStatusPersistenceService fileStatusPersistenceService; - public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier, FileModel fileModel) { + @Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log") + public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) { log.info("Merging EntityLog"); - List allManualChanges = allManualChanges(manualRedactions); - ManualRedactions manualRedactionsWhichRequireExtraInformation = ManualRedactions.builder() - .entriesToAdd(new HashSet<>()) - .resizeRedactions(new HashSet<>()) - .build(); + List allManualChangesExceptAdds = allManualChangesExceptAdds(manualRedactions); + + manualRedactions.getEntriesToAdd().forEach(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier)); // Sort manual changes by date, so we process them in order of when they were requested - allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList(); - allManualChanges.forEach(manualChange -> { - // this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher - if (manualChange instanceof ManualRedactionEntry) { - mergeManualRedactionEntries((ManualRedactionEntry) manualChange, entityLog, dossier, manualRedactionsWhichRequireExtraInformation); - } else if (manualChange instanceof IdRemoval) { - mergeIdsToRemove((IdRemoval) manualChange, entityLog); - } else if (manualChange instanceof ManualResizeRedaction) { - mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog, manualRedactionsWhichRequireExtraInformation); - } else if (manualChange instanceof ManualLegalBasisChange) { - mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog); - } else if (manualChange instanceof ManualRecategorization) { - mergeRecategorizations((ManualRecategorization) manualChange, entityLog, dossier); - } else if (manualChange instanceof ManualForceRedaction) { - mergeForceRedactions((ManualForceRedaction) manualChange, entityLog); + allManualChangesExceptAdds = allManualChangesExceptAdds.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).collect(Collectors.toList()); + final int analysisNumber = entityLog.getAnalysisNumber(); + if (!allManualChangesExceptAdds.isEmpty()) { + for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) { + var optionalManualChange = allManualChangesExceptAdds.stream().filter(manualChange -> manualChange.getAnnotationId().equals(entityLogEntry.getId())).findAny(); + if (optionalManualChange.isPresent()) { + var manualChange = optionalManualChange.get(); + if (manualChange instanceof IdRemoval idRemoval) { + mergeIdsToRemove(idRemoval, entityLogEntry, analysisNumber); + } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) { + mergeResizeRedactions(manualResizeRedaction, entityLogEntry, analysisNumber); + } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) { + mergeLegalBasisChanges(manualLegalBasisChange, entityLogEntry, analysisNumber); + } else if (manualChange instanceof ManualRecategorization manualRecategorization) { + mergeRecategorizations(manualRecategorization, entityLogEntry, dossier, analysisNumber); + } else if (manualChange instanceof ManualForceRedaction manualForceRedaction) { + mergeForceRedactions(manualForceRedaction, entityLogEntry, analysisNumber); + } + allManualChangesExceptAdds.remove(manualChange); + } } - }); - - if (!manualRedactionsWhichRequireExtraInformation.getEntriesToAdd().isEmpty() || !manualRedactionsWhichRequireExtraInformation.getResizeRedactions().isEmpty()) { - sendToAnalyseQueue(fileModel.getId(), dossier, fileModel, manualRedactionsWhichRequireExtraInformation); } log.info("EntityLog merged successfully."); @@ -98,7 +98,7 @@ public class EntityLogMergeService { } - private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier, ManualRedactions manualRedactions) { + private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) { if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) { return; @@ -108,7 +108,7 @@ public class EntityLogMergeService { changes.add(Change.builder() .analysisNumber(entityLog.getAnalysisNumber()) - .dateTime(OffsetDateTime.now()) + .dateTime(manualRedactionEntry.getRequestDate()) .type(ChangeType.ADDED) .build()); @@ -118,9 +118,7 @@ public class EntityLogMergeService { var matchingEntities = entityLog.getEntityLogEntry().stream() .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0))) .toList(); - matchingEntities.forEach(matchingEntity -> { - mergeFalsePositive(entityLog, matchingEntity); - }); + matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity)); return; } @@ -154,8 +152,6 @@ public class EntityLogMergeService { .reference(new HashSet<>()) .importedRedactionIntersections(new HashSet<>()) .build()); - - manualRedactions.getEntriesToAdd().add(manualRedactionEntry); } @@ -183,25 +179,19 @@ public class EntityLogMergeService { - private void mergeIdsToRemove(IdRemoval idRemoval, EntityLog entityLog) { + private void mergeIdsToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setState(EntryState.IGNORED); - addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber()); - entity.get().getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) - .requestedDate(idRemoval.getRequestDate()) - .processedDate(null) - .userId(idRemoval.getUser()).build()); - } + entityLogEntry.setState(EntryState.IGNORED); + addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate()); + entityLogEntry.getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) + .requestedDate(idRemoval.getRequestDate()) + .processedDate(null) + .userId(idRemoval.getUser()).build()); } - private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog, ManualRedactions manualRedactions) { + private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny(); - if (entity.isPresent()) { - EntityLogEntry entityLogEntry = entity.get(); entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter()); entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore()); entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions())); @@ -215,57 +205,48 @@ public class EntityLogMergeService { manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue())); } entityLogEntry.getManualChanges().add(manualChange.build()); - - manualRedactions.getResizeRedactions().add(manualResizeRedaction); } } - private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLog entityLog) { + private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis()); - entity.get().setSection(manualLegalBasisChange.getSection()); - entity.get().setValue(manualLegalBasisChange.getValue()); - addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - Map propertyChanges = new HashMap<>(); - if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) { - propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis()); - } - if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) { - propertyChanges.put("value", manualLegalBasisChange.getValue()); - } - if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) { - propertyChanges.put("section", manualLegalBasisChange.getSection()); - } - entity.get().getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) - .requestedDate(manualLegalBasisChange.getRequestDate()) - .processedDate(null) - .propertyChanges(propertyChanges) - .userId(manualLegalBasisChange.getUser()) - .build()); + entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + entityLogEntry.setSection(manualLegalBasisChange.getSection()); + entityLogEntry.setValue(manualLegalBasisChange.getValue()); + addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate()); + Map propertyChanges = new HashMap<>(); + if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) { + propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis()); } + if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) { + propertyChanges.put("value", manualLegalBasisChange.getValue()); + } + if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) { + propertyChanges.put("section", manualLegalBasisChange.getSection()); + } + entityLogEntry.getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) + .requestedDate(manualLegalBasisChange.getRequestDate()) + .processedDate(null) + .propertyChanges(propertyChanges) + .userId(manualLegalBasisChange.getUser()) + .build()); } - private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) { + private void mergeRecategorizations(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) { boolean isHint = isHint(recategorization.getType(), dossier); - var optionalEntityLogEntry = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny(); - if (optionalEntityLogEntry.isPresent()) { - var entity = optionalEntityLogEntry.get(); - entity.setType(recategorization.getType()); - entity.setEntryType(getEntryType(isHint, recategorization.getType())); - entity.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); - addChanges(entity.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - entity.getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.RECATEGORIZE) - .requestedDate(recategorization.getRequestDate()) - .processedDate(recategorization.getProcessedDate()) - .userId(recategorization.getUser()) - .propertyChanges(Map.of("type", recategorization.getType())) - .build()); - } + entityLogEntry.setType(recategorization.getType()); + entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType())); + entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); + addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate()); + entityLogEntry.getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.RECATEGORIZE) + .requestedDate(recategorization.getRequestDate()) + .processedDate(recategorization.getProcessedDate()) + .userId(recategorization.getUser()) + .propertyChanges(Map.of("type", recategorization.getType())) + .build()); } @@ -287,31 +268,28 @@ public class EntityLogMergeService { } - private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) { + private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setLegalBasis(forceRedaction.getLegalBasis()); - entity.get().setState(EntryState.APPLIED); - addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - var forceRedactManualChange = ManualChange.builder() - .manualRedactionType(ManualRedactionType.FORCE_REDACT) - .requestedDate(forceRedaction.getRequestDate()) - .processedDate(forceRedaction.getProcessedDate()) - .userId(forceRedaction.getUser()); - if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { - forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); - } - entity.get().getManualChanges().add(forceRedactManualChange.build()); + entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis()); + entityLogEntry.setState(EntryState.APPLIED); + addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate()); + var forceRedactManualChange = ManualChange.builder() + .manualRedactionType(ManualRedactionType.FORCE_REDACT) + .requestedDate(forceRedaction.getRequestDate()) + .processedDate(forceRedaction.getProcessedDate()) + .userId(forceRedaction.getUser()); + if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { + forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); } + entityLogEntry.getManualChanges().add(forceRedactManualChange.build()); } - private void addChanges(List changes, ChangeType changeType, int analysisNumber) { + private void addChanges(List changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) { if (!changes.isEmpty()) { changes.add(Change.builder() .analysisNumber(analysisNumber + 1) - .dateTime(OffsetDateTime.now()) + .dateTime(offsetDateTime) .type(changeType) .build()); } else { @@ -349,17 +327,16 @@ public class EntityLogMergeService { return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList()); } - private List allManualChanges(ManualRedactions manualRedactions) { + private List allManualChangesExceptAdds(ManualRedactions manualRedactions) { - return Stream.of(manualRedactions.getEntriesToAdd(), - manualRedactions.getForceRedactions(), + return Stream.of(manualRedactions.getForceRedactions(), manualRedactions.getResizeRedactions(), manualRedactions.getRecategorizations(), manualRedactions.getIdsToRemove(), manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); } - private void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) { + public void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) { var fileEntity = fileStatusPersistenceService.getStatus(fileId); var analyseRequest = AnalyzeRequest.builder() @@ -372,7 +349,6 @@ public class EntityLogMergeService { .dossierTemplateId(dossier.getDossierTemplateId()) .lastProcessed(fileModel.getLastProcessed()) .fileAttributes(fileStatusService.convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) - .excludedPages(fileModel.getExcludedPages()) .build(); log.info("Sending Surrounding Text Analysis for unprocessed manual redactions: {} for file: {} ", manualRedactions, fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java index 75491e9a3..17e6c70a2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java @@ -15,8 +15,10 @@ import com.iqser.red.service.persistence.management.v1.processor.service.manualr import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @@ -37,12 +39,14 @@ public class EntityLogService { EntityLogMergeService entityLogMergeService; + @Observed(name = "EntityLogService", contextualName = "get-entity-log") public EntityLog getEntityLog(String dossierId, String fileId) { return getEntityLog(dossierId, fileId, Collections.emptyList(), false); } + @Observed(name = "EntityLogService", contextualName = "get-entity-log") public EntityLog getEntityLog(String dossierId, String fileId, List excludedTypes, boolean includeUnprocessed) { var fileStatus = fileStatusService.getStatus(fileId); @@ -62,7 +66,13 @@ public class EntityLogService { if (includeUnprocessed) { DossierEntity dossier = dossierService.getDossierById(dossierId); ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true); - entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier, fileStatus); + entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier); + } + + if (fileStatus.getExcludedPages() != null && !fileStatus.getExcludedPages().isEmpty()) { + entityLog.getEntityLogEntry() + .forEach(entry -> entry.setExcluded(entry.getPositions().stream().anyMatch(position -> fileStatus.getExcludedPages().contains(position.getPageNumber())) // + && entry.getManualChanges().stream().noneMatch(m -> m.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY)))); } Map commentCountPerAnnotationId = commentService.getCommentCounts(fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index e3e09e7a8..385238d2b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -217,7 +217,6 @@ public class FileStatusService { .dossierTemplateId(dossier.getDossierTemplateId()) .lastProcessed(fileModel.getLastProcessed()) .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) - .excludedPages(fileModel.getExcludedPages()) .build(); log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index cc08cba8c..65e583478 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -71,7 +71,7 @@ public class DownloadPreparationService { downloadStatus.getFiles().forEach(fileEntity -> { - RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).includeUnprocessed(reportResultMessage.getIncludeUnprocessed()).build(); + RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build(); log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId()); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message); }); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java index 74d080583..fe26f20b5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java @@ -4,12 +4,9 @@ import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; -import jakarta.transaction.Transactional; - import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; @@ -21,6 +18,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage; +import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; @Service @@ -52,7 +50,6 @@ public class DownloadProcessorService { .downloadId(downloadJob.getStorageId()) .dossierId(dossier.getId()) .dossierTemplateId(dossierService.getDossierById(dossier.getId()).getDossierTemplateId()) - .includeUnprocessed(downloadJob.getIncludeUnprocessed()) .fileIds(filenameSortedFileIds) .templateIds(downloadStatus.getReports().stream().map(ReportTemplateEntity::getTemplateId).collect(Collectors.toSet())) .build(), 1); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java index d253bc4ac..fde6b1509 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java @@ -2,19 +2,23 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService; import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; -import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; @@ -35,6 +39,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -59,17 +66,18 @@ public class ManualRedactionService { ManualRedactionProviderService manualRedactionProviderService; AnalysisFlagsCalculationService analysisFlagsCalculationService; EntityLogService entityLogService; - DictionaryManagementService dictionaryManagementService; HashFunction hashFunction = Hashing.murmur3_128(); ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler; + EntityLogMergeService entityLogMergeService; @Transactional public List addAddRedaction(String dossierId, String fileId, List addRedactionRequests) { var response = new ArrayList(); + Set manualRedactionEntryEntities = new HashSet<>(); - dossierPersistenceService.getAndValidateDossier(dossierId); + var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId); for (AddRedactionRequest addRedactionRequest : addRedactionRequests) { manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue()); @@ -77,7 +85,7 @@ public class ManualRedactionService { String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); - addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); + manualRedactionEntryEntities.add(addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest)); Set typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, addRedactionRequest.getValue(), @@ -94,6 +102,17 @@ public class ManualRedactionService { response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } + manualRedactionEntryEntities = manualRedactionEntryEntities.stream() + .filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) + .collect(Collectors.toSet()); + + if (!manualRedactionEntryEntities.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder() + .entriesToAdd(MagicConverter.convert(manualRedactionEntryEntities, ManualRedactionEntry.class)) + .build(); + entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions); + } + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); reprocess(dossierId, fileId); @@ -233,12 +252,13 @@ public class ManualRedactionService { public List addResizeRedaction(String dossierId, String fileId, List resizeRedactionRequests) { List response = new ArrayList<>(); - + Set manualResizeRedactionEntities = new HashSet<>(); EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId); for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) { var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest); + manualResizeRedactionEntities.add(resizeRedaction); if (resizeRedactionRequest.getComment() != null) { Long commentId = commentService.addCommentAndGetId(fileId, @@ -259,6 +279,19 @@ public class ManualRedactionService { typeIdsOfModifiedDictionaries); } + manualResizeRedactionEntities = manualResizeRedactionEntities.stream() + .filter(manualRedactionEntry -> !manualRedactionEntry.getUpdateDictionary()) + .collect(Collectors.toSet()); + + if (!manualResizeRedactionEntities.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder() + .resizeRedactions(MagicConverter.convert(manualResizeRedactionEntities, ManualResizeRedaction.class)) + .build(); + entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + reprocess(dossierId, fileId); analysisFlagsCalculationService.calculateFlags(dossierId, fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java index 2cd6f3f05..a9dac6693 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java @@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -26,12 +27,14 @@ public class CvAnalysisMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusService fileStatusService; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObservationRegistry observationRegistry; @SneakyThrows @RabbitListener(queues = MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE) public void receive(CvAnalysisServiceResponse response) { + addFileIdToTrace(response.getFileId()); fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false); log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE, response.getDossierId(), response.getFileId()); @@ -43,7 +46,7 @@ public class CvAnalysisMessageReceiver { public void handleDLQMessage(Message failedMessage) { var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class); - + addFileIdToTrace(response.getFileId()); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId()); fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1", OffsetDateTime.now().truncatedTo( @@ -51,4 +54,12 @@ public class CvAnalysisMessageReceiver { } + // This can be removed if tracing is implemented in python services. + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 00cd9ea22..30767fc32 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -31,6 +32,7 @@ public class ImageMessageReceiver { private final FileStatusService fileStatusService; private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObservationRegistry observationRegistry; @SneakyThrows @@ -41,6 +43,7 @@ public class ImageMessageReceiver { String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); + addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -55,10 +58,19 @@ public class ImageMessageReceiver { }); String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); + addFileIdToTrace(fileId); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); } + + // This can be removed if tracing is implemented in python services. + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java index 06150cde6..62feed4c6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java @@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -28,6 +29,7 @@ public class NerMessageReceiver { private final FileStatusService fileStatusService; private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; + private final ObservationRegistry observationRegistry; @SneakyThrows @@ -39,6 +41,7 @@ public class NerMessageReceiver { String dossierId = (String) entityResponse.get("dossierId"); String fileId = (String) entityResponse.get("fileId"); + addFileIdToTrace(fileId); log.info("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE, dossierId, fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -52,10 +55,19 @@ public class NerMessageReceiver { }); String dossierId = (String) entityResponse.get("dossierId"); String fileId = (String) entityResponse.get("fileId"); + addFileIdToTrace(fileId); log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); } + + // This can be removed if tracing is implemented in python services. + private void addFileIdToTrace(String fileId){ + if(observationRegistry.getCurrentObservation() != null){ + observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId); + } + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java index 9060d42ba..2eaa4b563 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java @@ -54,6 +54,7 @@ import io.micrometer.common.KeyValues; import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.observation.ObservationRegistry; +import io.micrometer.observation.aop.ObservedAspect; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -78,6 +79,13 @@ public class Application implements ApplicationContextAware { } + @Bean + public ObservedAspect observedAspect(ObservationRegistry observationRegistry) { + + return new ObservedAspect(observationRegistry); + } + + @Bean public TimedAspect timedAspect(MeterRegistry registry) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java index 366d4f3a8..3f30195c5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java @@ -4,22 +4,19 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; -import com.knecon.fforesight.tenantcommons.TenantProvider; -import com.knecon.fforesight.tenantcommons.model.TenantResponse; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.mock.web.MockMultipartFile; import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient; @@ -31,6 +28,8 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver; import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver; +import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; @@ -45,6 +44,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; import lombok.AccessLevel; import lombok.SneakyThrows; @@ -93,8 +93,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes DownloadStatusPersistenceService downloadStatusPersistenceService; - - @BeforeEach public void createTestData() { @@ -140,7 +138,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes ReportResultMessage reportResultMessage = new ReportResultMessage(); reportResultMessage.setUserId(USER_ID); reportResultMessage.setDownloadId(downloadId); - reportResultMessage.setIncludeUnprocessed(true); downloadReportMessageReceiver.receive(reportResultMessage); redactionResultMessageReceiver.receive(RedactionResultMessage.builder() diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java index c93708140..d81215b34 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java @@ -133,12 +133,7 @@ public class EntityLogMergeTest { EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder() .dossierTemplateId(dossierTemplateId) - .build(), FileModel.builder().id(fileId).build()); - - Mockito.verify(rabbitTemplate).convertAndSend(eq(MessagingConfiguration.REDACTION_PRIORITY_QUEUE), captor.capture()); - AnalyzeRequest analyzeRequest = captor.getValue(); - assertEquals(analyzeRequest.getManualRedactions().getEntriesToAdd().size(), 1); - assertEquals(analyzeRequest.getManualRedactions().getResizeRedactions().size(), 1); + .build()); assertNotNull(response); assertFalse(response.getEntityLogEntry().isEmpty()); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java index 56b51d754..b00739042 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java @@ -28,8 +28,6 @@ public class AnalyzeRequest { private OffsetDateTime lastProcessed; private int analysisNumber; - @Builder.Default - private Set excludedPages = new HashSet<>(); @Builder.Default private Set sectionsToReanalyse = new HashSet<>();