Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'master' into RED-7256-npe

Browse Source 
# Conflicts:
#	persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
This commit is contained in:
Corina Olariu 2023-12-20 11:07:11 +02:00
commit 2948ae1c10
16 changed files with 199 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts
View File

@ -7,8 +7,8 @@ plugins {
}
val redactionServiceVersion by rootProject.extra { "4.199.0" }
val pdftronRedactionServiceVersion by rootProject.extra { "4.38.0" }
val redactionReportServiceVersion by rootProject.extra { "4.36.0" }
val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" }
val redactionReportServiceVersion by rootProject.extra { "4.47.0" }
val searchServiceVersion by rootProject.extra { "2.71.0" }
repositories {

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/ACLBeanConfiguration.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.acl;
import com.knecon.fforesight.databasetenantcommons.providers.MultiTenantDataSource;
import lombok.RequiredArgsConstructor;
import org.springframework.cache.CacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -7,7 +9,6 @@ import org.springframework.context.annotation.Primary;
import org.springframework.security.access.expression.method.DefaultMethodSecurityExpressionHandler;
import org.springframework.security.access.expression.method.MethodSecurityExpressionHandler;
import org.springframework.security.acls.AclPermissionCacheOptimizer;
import org.springframework.security.acls.AclPermissionEvaluator;
import org.springframework.security.acls.domain.AclAuthorizationStrategy;
import org.springframework.security.acls.domain.ConsoleAuditLogger;
import org.springframework.security.acls.domain.DefaultPermissionFactory;
@ -20,10 +21,6 @@ import org.springframework.security.acls.jdbc.LookupStrategy;
import org.springframework.security.acls.model.PermissionGrantingStrategy;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import com.knecon.fforesight.databasetenantcommons.providers.MultiTenantDataSource;
import lombok.RequiredArgsConstructor;
@Configuration
@RequiredArgsConstructor
@EnableTransactionManagement
@ -81,7 +78,7 @@ public class ACLBeanConfiguration {
public MethodSecurityExpressionHandler defaultMethodSecurityExpressionHandler() {
DefaultMethodSecurityExpressionHandler expressionHandler = new DefaultMethodSecurityExpressionHandler();
AclPermissionEvaluator permissionEvaluator = new AclPermissionEvaluator(aclService());
CustomAclPermissionEvaluator permissionEvaluator = new CustomAclPermissionEvaluator(aclService());
permissionEvaluator.setPermissionFactory(permissionFactory());
expressionHandler.setPermissionEvaluator(permissionEvaluator);

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/CustomAclPermissionEvaluator.java
View File

@ -1,18 +1,16 @@
package com.iqser.red.service.persistence.management.v1.processor.acl;
import java.io.Serializable;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import org.springframework.security.acls.AclPermissionEvaluator;
import org.springframework.security.acls.model.AclService;
import org.springframework.security.acls.model.NotFoundException;
import org.springframework.security.core.Authentication;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import java.io.Serializable;
public class CustomAclPermissionEvaluator extends AclPermissionEvaluator {
public CustomAclPermissionEvaluator(AclService aclService) {
super(aclService);
}

198
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -61,36 +61,36 @@ public class EntityLogMergeService {
private final FileStatusPersistenceService fileStatusPersistenceService;
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier, FileModel fileModel) {
@Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log")
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) {
log.info("Merging EntityLog");
List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions);
ManualRedactions manualRedactionsWhichRequireExtraInformation = ManualRedactions.builder()
.entriesToAdd(new HashSet<>())
.resizeRedactions(new HashSet<>())
.build();
List<BaseAnnotation> allManualChangesExceptAdds = allManualChangesExceptAdds(manualRedactions);
manualRedactions.getEntriesToAdd().forEach(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier));
// Sort manual changes by date, so we process them in order of when they were requested
allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
allManualChanges.forEach(manualChange -> {
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
if (manualChange instanceof ManualRedactionEntry) {
mergeManualRedactionEntries((ManualRedactionEntry) manualChange, entityLog, dossier, manualRedactionsWhichRequireExtraInformation);
} else if (manualChange instanceof IdRemoval) {
mergeIdsToRemove((IdRemoval) manualChange, entityLog);
} else if (manualChange instanceof ManualResizeRedaction) {
mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog, manualRedactionsWhichRequireExtraInformation);
} else if (manualChange instanceof ManualLegalBasisChange) {
mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog);
} else if (manualChange instanceof ManualRecategorization) {
mergeRecategorizations((ManualRecategorization) manualChange, entityLog, dossier);
} else if (manualChange instanceof ManualForceRedaction) {
mergeForceRedactions((ManualForceRedaction) manualChange, entityLog);
allManualChangesExceptAdds = allManualChangesExceptAdds.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).collect(Collectors.toList());
final int analysisNumber = entityLog.getAnalysisNumber();
if (!allManualChangesExceptAdds.isEmpty()) {
for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) {
var optionalManualChange = allManualChangesExceptAdds.stream().filter(manualChange -> manualChange.getAnnotationId().equals(entityLogEntry.getId())).findAny();
if (optionalManualChange.isPresent()) {
var manualChange = optionalManualChange.get();
if (manualChange instanceof IdRemoval idRemoval) {
mergeIdsToRemove(idRemoval, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
mergeResizeRedactions(manualResizeRedaction, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
mergeLegalBasisChanges(manualLegalBasisChange, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualRecategorization manualRecategorization) {
mergeRecategorizations(manualRecategorization, entityLogEntry, dossier, analysisNumber);
} else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
mergeForceRedactions(manualForceRedaction, entityLogEntry, analysisNumber);
}
allManualChangesExceptAdds.remove(manualChange);
}
}
});
if (!manualRedactionsWhichRequireExtraInformation.getEntriesToAdd().isEmpty() || !manualRedactionsWhichRequireExtraInformation.getResizeRedactions().isEmpty()) {
sendToAnalyseQueue(fileModel.getId(), dossier, fileModel, manualRedactionsWhichRequireExtraInformation);
}
log.info("EntityLog merged successfully.");
@ -98,7 +98,7 @@ public class EntityLogMergeService {
}
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier, ManualRedactions manualRedactions) {
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) {
if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) {
return;
@ -108,7 +108,7 @@ public class EntityLogMergeService {
changes.add(Change.builder()
.analysisNumber(entityLog.getAnalysisNumber())
.dateTime(OffsetDateTime.now())
.dateTime(manualRedactionEntry.getRequestDate())
.type(ChangeType.ADDED)
.build());
@ -118,9 +118,7 @@ public class EntityLogMergeService {
var matchingEntities = entityLog.getEntityLogEntry().stream()
.filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0)))
.toList();
matchingEntities.forEach(matchingEntity -> {
mergeFalsePositive(entityLog, matchingEntity);
});
matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity));
return;
}
@ -154,8 +152,6 @@ public class EntityLogMergeService {
.reference(new HashSet<>())
.importedRedactionIntersections(new HashSet<>())
.build());
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
}
@ -183,25 +179,19 @@ public class EntityLogMergeService {
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLog entityLog) {
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setState(EntryState.IGNORED);
addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber());
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(null)
.userId(idRemoval.getUser()).build());
}
entityLogEntry.setState(EntryState.IGNORED);
addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(null)
.userId(idRemoval.getUser()).build());
}
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog, ManualRedactions manualRedactions) {
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
EntityLogEntry entityLogEntry = entity.get();
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
@ -215,57 +205,48 @@ public class EntityLogMergeService {
manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
}
entityLogEntry.getManualChanges().add(manualChange.build());
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
}
}
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLog entityLog) {
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis());
entity.get().setSection(manualLegalBasisChange.getSection());
entity.get().setValue(manualLegalBasisChange.getValue());
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
Map<String, String> propertyChanges = new HashMap<>();
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
}
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) {
propertyChanges.put("value", manualLegalBasisChange.getValue());
}
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection());
}
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
Map<String, String> propertyChanges = new HashMap<>();
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
}
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) {
propertyChanges.put("value", manualLegalBasisChange.getValue());
}
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection());
}
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
}
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) {
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
boolean isHint = isHint(recategorization.getType(), dossier);
var optionalEntityLogEntry = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
if (optionalEntityLogEntry.isPresent()) {
var entity = optionalEntityLogEntry.get();
entity.setType(recategorization.getType());
entity.setEntryType(getEntryType(isHint, recategorization.getType()));
entity.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entity.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entity.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
.requestedDate(recategorization.getRequestDate())
.processedDate(recategorization.getProcessedDate())
.userId(recategorization.getUser())
.propertyChanges(Map.of("type", recategorization.getType()))
.build());
}
entityLogEntry.setType(recategorization.getType());
entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
.requestedDate(recategorization.getRequestDate())
.processedDate(recategorization.getProcessedDate())
.userId(recategorization.getUser())
.propertyChanges(Map.of("type", recategorization.getType()))
.build());
}
@ -287,31 +268,28 @@ public class EntityLogMergeService {
}
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) {
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setLegalBasis(forceRedaction.getLegalBasis());
entity.get().setState(EntryState.APPLIED);
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entity.get().getManualChanges().add(forceRedactManualChange.build());
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
}
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber) {
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) {
changes.add(Change.builder()
.analysisNumber(analysisNumber + 1)
.dateTime(OffsetDateTime.now())
.dateTime(offsetDateTime)
.type(changeType)
.build());
} else {
@ -349,17 +327,16 @@ public class EntityLogMergeService {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
}
private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) {
private List<BaseAnnotation> allManualChangesExceptAdds(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getEntriesToAdd(),
manualRedactions.getForceRedactions(),
return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
}
private void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
public void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
var analyseRequest = AnalyzeRequest.builder()
@ -372,7 +349,6 @@ public class EntityLogMergeService {
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed())
.fileAttributes(fileStatusService.convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build();
log.info("Sending Surrounding Text Analysis for unprocessed manual redactions: {} for file: {} ", manualRedactions, fileId);

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java
View File

@ -15,8 +15,10 @@ import com.iqser.red.service.persistence.management.v1.processor.service.manualr
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@ -37,12 +39,14 @@ public class EntityLogService {
EntityLogMergeService entityLogMergeService;
@Observed(name = "EntityLogService", contextualName = "get-entity-log")
public EntityLog getEntityLog(String dossierId, String fileId) {
return getEntityLog(dossierId, fileId, Collections.emptyList(), false);
}
@Observed(name = "EntityLogService", contextualName = "get-entity-log")
public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes, boolean includeUnprocessed) {
var fileStatus = fileStatusService.getStatus(fileId);
@ -62,7 +66,13 @@ public class EntityLogService {
if (includeUnprocessed) {
DossierEntity dossier = dossierService.getDossierById(dossierId);
ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier, fileStatus);
entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier);
}
if (fileStatus.getExcludedPages() != null && !fileStatus.getExcludedPages().isEmpty()) {
entityLog.getEntityLogEntry()
.forEach(entry -> entry.setExcluded(entry.getPositions().stream().anyMatch(position -> fileStatus.getExcludedPages().contains(position.getPageNumber())) //
&& entry.getManualChanges().stream().noneMatch(m -> m.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY))));
}
Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -217,7 +217,6 @@ public class FileStatusService {
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed())
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build();
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -71,7 +71,7 @@ public class DownloadPreparationService {
downloadStatus.getFiles().forEach(fileEntity -> {
RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).includeUnprocessed(reportResultMessage.getIncludeUnprocessed()).build();
RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build();
log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId());
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message);
});

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java
View File

@ -4,12 +4,9 @@ import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -21,6 +18,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
@Service
@ -52,7 +50,6 @@ public class DownloadProcessorService {
.downloadId(downloadJob.getStorageId())
.dossierId(dossier.getId())
.dossierTemplateId(dossierService.getDossierById(dossier.getId()).getDossierTemplateId())
.includeUnprocessed(downloadJob.getIncludeUnprocessed())
.fileIds(filenameSortedFileIds)
.templateIds(downloadStatus.getReports().stream().map(ReportTemplateEntity::getTemplateId).collect(Collectors.toSet()))
.build(), 1);

43
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -2,19 +2,23 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
@ -35,6 +39,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -59,17 +66,18 @@ public class ManualRedactionService {
ManualRedactionProviderService manualRedactionProviderService;
AnalysisFlagsCalculationService analysisFlagsCalculationService;
EntityLogService entityLogService;
DictionaryManagementService dictionaryManagementService;
HashFunction hashFunction = Hashing.murmur3_128();
ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
EntityLogMergeService entityLogMergeService;
@Transactional
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
Set<ManualRedactionEntryEntity> manualRedactionEntryEntities = new HashSet<>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId);
for (AddRedactionRequest addRedactionRequest : addRedactionRequests) {
manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue());
@ -77,7 +85,7 @@ public class ManualRedactionService {
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
manualRedactionEntryEntities.add(addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest));
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
addRedactionRequest.getValue(),
@ -94,6 +102,17 @@ public class ManualRedactionService {
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
manualRedactionEntryEntities = manualRedactionEntryEntities.stream()
.filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
.collect(Collectors.toSet());
if (!manualRedactionEntryEntities.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(MagicConverter.convert(manualRedactionEntryEntities, ManualRedactionEntry.class))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
reprocess(dossierId, fileId);
@ -233,12 +252,13 @@ public class ManualRedactionService {
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) {
List<ManualAddResponse> response = new ArrayList<>();
Set<ManualResizeRedactionEntity> manualResizeRedactionEntities = new HashSet<>();
EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId);
for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
manualResizeRedactionEntities.add(resizeRedaction);
if (resizeRedactionRequest.getComment() != null) {
Long commentId = commentService.addCommentAndGetId(fileId,
@ -259,6 +279,19 @@ public class ManualRedactionService {
typeIdsOfModifiedDictionaries);
}
manualResizeRedactionEntities = manualResizeRedactionEntities.stream()
.filter(manualRedactionEntry -> !manualRedactionEntry.getUpdateDictionary())
.collect(Collectors.toSet());
if (!manualResizeRedactionEntities.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder()
.resizeRedactions(MagicConverter.convert(manualResizeRedactionEntities, ManualResizeRedaction.class))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
reprocess(dossierId, fileId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java
View File

@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -26,12 +27,14 @@ public class CvAnalysisMessageReceiver {
private final ObjectMapper objectMapper;
private final FileStatusService fileStatusService;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows
@RabbitListener(queues = MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE)
public void receive(CvAnalysisServiceResponse response) {
addFileIdToTrace(response.getFileId());
fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false);
log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE, response.getDossierId(), response.getFileId());
@ -43,7 +46,7 @@ public class CvAnalysisMessageReceiver {
public void handleDLQMessage(Message failedMessage) {
var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class);
addFileIdToTrace(response.getFileId());
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId());
fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1", OffsetDateTime.now().truncatedTo(
@ -51,4 +54,12 @@ public class CvAnalysisMessageReceiver {
}
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -31,6 +32,7 @@ public class ImageMessageReceiver {
private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows
@ -41,6 +43,7 @@ public class ImageMessageReceiver {
String dossierId = imageResponse.path("dossierId").asText();
String fileId = imageResponse.path("fileId").asText();
addFileIdToTrace(fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
@ -55,10 +58,19 @@ public class ImageMessageReceiver {
});
String dossierId = (String) imageResponse.get("dossierId");
String fileId = (String) imageResponse.get("fileId");
addFileIdToTrace(fileId);
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId);
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
}
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java
View File

@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -28,6 +29,7 @@ public class NerMessageReceiver {
private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows
@ -39,6 +41,7 @@ public class NerMessageReceiver {
String dossierId = (String) entityResponse.get("dossierId");
String fileId = (String) entityResponse.get("fileId");
addFileIdToTrace(fileId);
log.info("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE, dossierId, fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
@ -52,10 +55,19 @@ public class NerMessageReceiver {
});
String dossierId = (String) entityResponse.get("dossierId");
String fileId = (String) entityResponse.get("fileId");
addFileIdToTrace(fileId);
log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId);
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId,
new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
}
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

8
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -54,6 +54,7 @@ import io.micrometer.common.KeyValues;
import io.micrometer.core.aop.TimedAspect;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.aop.ObservedAspect;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -78,6 +79,13 @@ public class Application implements ApplicationContextAware {
}
@Bean
public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
return new ObservedAspect(observationRegistry);
}
@Bean
public TimedAspect timedAspect(MeterRegistry registry) {

19
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
View File

@ -4,22 +4,19 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.mock.web.MockMultipartFile;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
@ -31,6 +28,8 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
@ -45,6 +44,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import lombok.AccessLevel;
import lombok.SneakyThrows;
@ -93,8 +93,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
DownloadStatusPersistenceService downloadStatusPersistenceService;
@BeforeEach
public void createTestData() {
@ -140,7 +138,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
ReportResultMessage reportResultMessage = new ReportResultMessage();
reportResultMessage.setUserId(USER_ID);
reportResultMessage.setDownloadId(downloadId);
reportResultMessage.setIncludeUnprocessed(true);
downloadReportMessageReceiver.receive(reportResultMessage);
redactionResultMessageReceiver.receive(RedactionResultMessage.builder()

7
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java
View File

@ -133,12 +133,7 @@ public class EntityLogMergeTest {
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder()
.dossierTemplateId(dossierTemplateId)
.build(), FileModel.builder().id(fileId).build());
Mockito.verify(rabbitTemplate).convertAndSend(eq(MessagingConfiguration.REDACTION_PRIORITY_QUEUE), captor.capture());
AnalyzeRequest analyzeRequest = captor.getValue();
assertEquals(analyzeRequest.getManualRedactions().getEntriesToAdd().size(), 1);
assertEquals(analyzeRequest.getManualRedactions().getResizeRedactions().size(), 1);
.build());
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java
View File

@ -28,8 +28,6 @@ public class AnalyzeRequest {
private OffsetDateTime lastProcessed;
private int analysisNumber;
@Builder.Default
private Set<Integer> excludedPages = new HashSet<>();
@Builder.Default
private Set<Integer> sectionsToReanalyse = new HashSet<>();