Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'master' into RED-7256-npe

Browse Source 
# Conflicts:
#	persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
This commit is contained in:
Corina Olariu 2023-12-20 11:07:11 +02:00
commit 2948ae1c10
16 changed files with 199 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts
View File

@ -7,8 +7,8 @@ plugins {
} }
val redactionServiceVersion by rootProject.extra { "4.199.0" } val redactionServiceVersion by rootProject.extra { "4.199.0" }
val pdftronRedactionServiceVersion by rootProject.extra { "4.38.0" } val pdftronRedactionServiceVersion by rootProject.extra { "4.48.0" }
val redactionReportServiceVersion by rootProject.extra { "4.36.0" } val redactionReportServiceVersion by rootProject.extra { "4.47.0" }
val searchServiceVersion by rootProject.extra { "2.71.0" } val searchServiceVersion by rootProject.extra { "2.71.0" }
repositories { repositories {

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/ACLBeanConfiguration.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.acl; package com.iqser.red.service.persistence.management.v1.processor.acl;
import com.knecon.fforesight.databasetenantcommons.providers.MultiTenantDataSource;
import lombok.RequiredArgsConstructor;
import org.springframework.cache.CacheManager; import org.springframework.cache.CacheManager;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@ -7,7 +9,6 @@ import org.springframework.context.annotation.Primary;
import org.springframework.security.access.expression.method.DefaultMethodSecurityExpressionHandler; import org.springframework.security.access.expression.method.DefaultMethodSecurityExpressionHandler;
import org.springframework.security.access.expression.method.MethodSecurityExpressionHandler; import org.springframework.security.access.expression.method.MethodSecurityExpressionHandler;
import org.springframework.security.acls.AclPermissionCacheOptimizer; import org.springframework.security.acls.AclPermissionCacheOptimizer;
import org.springframework.security.acls.AclPermissionEvaluator;
import org.springframework.security.acls.domain.AclAuthorizationStrategy; import org.springframework.security.acls.domain.AclAuthorizationStrategy;
import org.springframework.security.acls.domain.ConsoleAuditLogger; import org.springframework.security.acls.domain.ConsoleAuditLogger;
import org.springframework.security.acls.domain.DefaultPermissionFactory; import org.springframework.security.acls.domain.DefaultPermissionFactory;
@ -20,10 +21,6 @@ import org.springframework.security.acls.jdbc.LookupStrategy;
import org.springframework.security.acls.model.PermissionGrantingStrategy; import org.springframework.security.acls.model.PermissionGrantingStrategy;
import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.EnableTransactionManagement;
import com.knecon.fforesight.databasetenantcommons.providers.MultiTenantDataSource;
import lombok.RequiredArgsConstructor;
@Configuration @Configuration
@RequiredArgsConstructor @RequiredArgsConstructor
@EnableTransactionManagement @EnableTransactionManagement
@ -81,7 +78,7 @@ public class ACLBeanConfiguration {
public MethodSecurityExpressionHandler defaultMethodSecurityExpressionHandler() { public MethodSecurityExpressionHandler defaultMethodSecurityExpressionHandler() {
DefaultMethodSecurityExpressionHandler expressionHandler = new DefaultMethodSecurityExpressionHandler(); DefaultMethodSecurityExpressionHandler expressionHandler = new DefaultMethodSecurityExpressionHandler();
AclPermissionEvaluator permissionEvaluator = new AclPermissionEvaluator(aclService()); CustomAclPermissionEvaluator permissionEvaluator = new CustomAclPermissionEvaluator(aclService());
permissionEvaluator.setPermissionFactory(permissionFactory()); permissionEvaluator.setPermissionFactory(permissionFactory());
expressionHandler.setPermissionEvaluator(permissionEvaluator); expressionHandler.setPermissionEvaluator(permissionEvaluator);

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/acl/CustomAclPermissionEvaluator.java
View File

@ -1,18 +1,16 @@
package com.iqser.red.service.persistence.management.v1.processor.acl; package com.iqser.red.service.persistence.management.v1.processor.acl;
import java.io.Serializable; import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import org.springframework.security.acls.AclPermissionEvaluator; import org.springframework.security.acls.AclPermissionEvaluator;
import org.springframework.security.acls.model.AclService; import org.springframework.security.acls.model.AclService;
import org.springframework.security.acls.model.NotFoundException;
import org.springframework.security.core.Authentication; import org.springframework.security.core.Authentication;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; import java.io.Serializable;
public class CustomAclPermissionEvaluator extends AclPermissionEvaluator { public class CustomAclPermissionEvaluator extends AclPermissionEvaluator {
public CustomAclPermissionEvaluator(AclService aclService) { public CustomAclPermissionEvaluator(AclService aclService) {
super(aclService); super(aclService);
} }

198
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -61,36 +61,36 @@ public class EntityLogMergeService {
private final FileStatusPersistenceService fileStatusPersistenceService; private final FileStatusPersistenceService fileStatusPersistenceService;
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier, FileModel fileModel) { @Observed(name = "EntityLogMergeService", contextualName = "merge-entity-log")
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier) {
log.info("Merging EntityLog"); log.info("Merging EntityLog");
List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions); List<BaseAnnotation> allManualChangesExceptAdds = allManualChangesExceptAdds(manualRedactions);
ManualRedactions manualRedactionsWhichRequireExtraInformation = ManualRedactions.builder()
.entriesToAdd(new HashSet<>()) manualRedactions.getEntriesToAdd().forEach(manualRedactionEntry -> mergeManualRedactionEntries(manualRedactionEntry, entityLog, dossier));
.resizeRedactions(new HashSet<>())
.build();
// Sort manual changes by date, so we process them in order of when they were requested // Sort manual changes by date, so we process them in order of when they were requested
allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList(); allManualChangesExceptAdds = allManualChangesExceptAdds.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).collect(Collectors.toList());
allManualChanges.forEach(manualChange -> { final int analysisNumber = entityLog.getAnalysisNumber();
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher if (!allManualChangesExceptAdds.isEmpty()) {
if (manualChange instanceof ManualRedactionEntry) { for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) {
mergeManualRedactionEntries((ManualRedactionEntry) manualChange, entityLog, dossier, manualRedactionsWhichRequireExtraInformation); var optionalManualChange = allManualChangesExceptAdds.stream().filter(manualChange -> manualChange.getAnnotationId().equals(entityLogEntry.getId())).findAny();
} else if (manualChange instanceof IdRemoval) { if (optionalManualChange.isPresent()) {
mergeIdsToRemove((IdRemoval) manualChange, entityLog); var manualChange = optionalManualChange.get();
} else if (manualChange instanceof ManualResizeRedaction) { if (manualChange instanceof IdRemoval idRemoval) {
mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog, manualRedactionsWhichRequireExtraInformation); mergeIdsToRemove(idRemoval, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualLegalBasisChange) { } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog); mergeResizeRedactions(manualResizeRedaction, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualRecategorization) { } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
mergeRecategorizations((ManualRecategorization) manualChange, entityLog, dossier); mergeLegalBasisChanges(manualLegalBasisChange, entityLogEntry, analysisNumber);
} else if (manualChange instanceof ManualForceRedaction) { } else if (manualChange instanceof ManualRecategorization manualRecategorization) {
mergeForceRedactions((ManualForceRedaction) manualChange, entityLog); mergeRecategorizations(manualRecategorization, entityLogEntry, dossier, analysisNumber);
} else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
mergeForceRedactions(manualForceRedaction, entityLogEntry, analysisNumber);
}
allManualChangesExceptAdds.remove(manualChange);
}
} }
});
if (!manualRedactionsWhichRequireExtraInformation.getEntriesToAdd().isEmpty() || !manualRedactionsWhichRequireExtraInformation.getResizeRedactions().isEmpty()) {
sendToAnalyseQueue(fileModel.getId(), dossier, fileModel, manualRedactionsWhichRequireExtraInformation);
} }
log.info("EntityLog merged successfully."); log.info("EntityLog merged successfully.");
@ -98,7 +98,7 @@ public class EntityLogMergeService {
} }
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier, ManualRedactions manualRedactions) { private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier) {
if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) { if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) {
return; return;
@ -108,7 +108,7 @@ public class EntityLogMergeService {
changes.add(Change.builder() changes.add(Change.builder()
.analysisNumber(entityLog.getAnalysisNumber()) .analysisNumber(entityLog.getAnalysisNumber())
.dateTime(OffsetDateTime.now()) .dateTime(manualRedactionEntry.getRequestDate())
.type(ChangeType.ADDED) .type(ChangeType.ADDED)
.build()); .build());
@ -118,9 +118,7 @@ public class EntityLogMergeService {
var matchingEntities = entityLog.getEntityLogEntry().stream() var matchingEntities = entityLog.getEntityLogEntry().stream()
.filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0))) .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0)))
.toList(); .toList();
matchingEntities.forEach(matchingEntity -> { matchingEntities.forEach(matchingEntity -> mergeFalsePositive(entityLog, matchingEntity));
mergeFalsePositive(entityLog, matchingEntity);
});
return; return;
} }
@ -154,8 +152,6 @@ public class EntityLogMergeService {
.reference(new HashSet<>()) .reference(new HashSet<>())
.importedRedactionIntersections(new HashSet<>()) .importedRedactionIntersections(new HashSet<>())
.build()); .build());
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
} }
@ -183,25 +179,19 @@ public class EntityLogMergeService {
private void mergeIdsToRemove(IdRemoval idRemoval, EntityLog entityLog) { private void mergeIdsToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny(); entityLogEntry.setState(EntryState.IGNORED);
if (entity.isPresent()) { addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
entity.get().setState(EntryState.IGNORED); entityLogEntry.getManualChanges().add(ManualChange.builder()
addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber()); .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
entity.get().getManualChanges().add(ManualChange.builder() .requestedDate(idRemoval.getRequestDate())
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) .processedDate(null)
.requestedDate(idRemoval.getRequestDate()) .userId(idRemoval.getUser()).build());
.processedDate(null)
.userId(idRemoval.getUser()).build());
}
} }
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog, ManualRedactions manualRedactions) { private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
EntityLogEntry entityLogEntry = entity.get();
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter()); entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore()); entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions())); entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
@ -215,57 +205,48 @@ public class EntityLogMergeService {
manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue())); manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
} }
entityLogEntry.getManualChanges().add(manualChange.build()); entityLogEntry.getManualChanges().add(manualChange.build());
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
} }
} }
private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLog entityLog) { private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny(); entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
if (entity.isPresent()) { entityLogEntry.setSection(manualLegalBasisChange.getSection());
entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis()); entityLogEntry.setValue(manualLegalBasisChange.getValue());
entity.get().setSection(manualLegalBasisChange.getSection()); addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
entity.get().setValue(manualLegalBasisChange.getValue()); Map<String, String> propertyChanges = new HashMap<>();
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
Map<String, String> propertyChanges = new HashMap<>(); propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
}
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) {
propertyChanges.put("value", manualLegalBasisChange.getValue());
}
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection());
}
entity.get().getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
} }
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) {
propertyChanges.put("value", manualLegalBasisChange.getValue());
}
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection());
}
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(null)
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
} }
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) { private void mergeRecategorizations(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
boolean isHint = isHint(recategorization.getType(), dossier); boolean isHint = isHint(recategorization.getType(), dossier);
var optionalEntityLogEntry = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny(); entityLogEntry.setType(recategorization.getType());
if (optionalEntityLogEntry.isPresent()) { entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
var entity = optionalEntityLogEntry.get(); entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
entity.setType(recategorization.getType()); addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate());
entity.setEntryType(getEntryType(isHint, recategorization.getType())); entityLogEntry.getManualChanges().add(ManualChange.builder()
entity.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); .manualRedactionType(ManualRedactionType.RECATEGORIZE)
addChanges(entity.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); .requestedDate(recategorization.getRequestDate())
entity.getManualChanges().add(ManualChange.builder() .processedDate(recategorization.getProcessedDate())
.manualRedactionType(ManualRedactionType.RECATEGORIZE) .userId(recategorization.getUser())
.requestedDate(recategorization.getRequestDate()) .propertyChanges(Map.of("type", recategorization.getType()))
.processedDate(recategorization.getProcessedDate()) .build());
.userId(recategorization.getUser())
.propertyChanges(Map.of("type", recategorization.getType()))
.build());
}
} }
@ -287,31 +268,28 @@ public class EntityLogMergeService {
} }
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) { private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny(); entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
if (entity.isPresent()) { entityLogEntry.setState(EntryState.APPLIED);
entity.get().setLegalBasis(forceRedaction.getLegalBasis()); addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
entity.get().setState(EntryState.APPLIED); var forceRedactManualChange = ManualChange.builder()
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); .manualRedactionType(ManualRedactionType.FORCE_REDACT)
var forceRedactManualChange = ManualChange.builder() .requestedDate(forceRedaction.getRequestDate())
.manualRedactionType(ManualRedactionType.FORCE_REDACT) .processedDate(forceRedaction.getProcessedDate())
.requestedDate(forceRedaction.getRequestDate()) .userId(forceRedaction.getUser());
.processedDate(forceRedaction.getProcessedDate()) if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
.userId(forceRedaction.getUser()); forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entity.get().getManualChanges().add(forceRedactManualChange.build());
} }
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
} }
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber) { private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) { if (!changes.isEmpty()) {
changes.add(Change.builder() changes.add(Change.builder()
.analysisNumber(analysisNumber + 1) .analysisNumber(analysisNumber + 1)
.dateTime(OffsetDateTime.now()) .dateTime(offsetDateTime)
.type(changeType) .type(changeType)
.build()); .build());
} else { } else {
@ -349,17 +327,16 @@ public class EntityLogMergeService {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList()); return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
} }
private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) { private List<BaseAnnotation> allManualChangesExceptAdds(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getEntriesToAdd(), return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(), manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(), manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(), manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
} }
private void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) { public void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId); var fileEntity = fileStatusPersistenceService.getStatus(fileId);
var analyseRequest = AnalyzeRequest.builder() var analyseRequest = AnalyzeRequest.builder()
@ -372,7 +349,6 @@ public class EntityLogMergeService {
.dossierTemplateId(dossier.getDossierTemplateId()) .dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed()) .lastProcessed(fileModel.getLastProcessed())
.fileAttributes(fileStatusService.convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) .fileAttributes(fileStatusService.convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build(); .build();
log.info("Sending Surrounding Text Analysis for unprocessed manual redactions: {} for file: {} ", manualRedactions, fileId); log.info("Sending Surrounding Text Analysis for unprocessed manual redactions: {} for file: {} ", manualRedactions, fileId);

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java
View File

@ -15,8 +15,10 @@ import com.iqser.red.service.persistence.management.v1.processor.service.manualr
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -37,12 +39,14 @@ public class EntityLogService {
EntityLogMergeService entityLogMergeService; EntityLogMergeService entityLogMergeService;
@Observed(name = "EntityLogService", contextualName = "get-entity-log")
public EntityLog getEntityLog(String dossierId, String fileId) { public EntityLog getEntityLog(String dossierId, String fileId) {
return getEntityLog(dossierId, fileId, Collections.emptyList(), false); return getEntityLog(dossierId, fileId, Collections.emptyList(), false);
} }
@Observed(name = "EntityLogService", contextualName = "get-entity-log")
public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes, boolean includeUnprocessed) { public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes, boolean includeUnprocessed) {
var fileStatus = fileStatusService.getStatus(fileId); var fileStatus = fileStatusService.getStatus(fileId);
@ -62,7 +66,13 @@ public class EntityLogService {
if (includeUnprocessed) { if (includeUnprocessed) {
DossierEntity dossier = dossierService.getDossierById(dossierId); DossierEntity dossier = dossierService.getDossierById(dossierId);
ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true); ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier, fileStatus); entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier);
}
if (fileStatus.getExcludedPages() != null && !fileStatus.getExcludedPages().isEmpty()) {
entityLog.getEntityLogEntry()
.forEach(entry -> entry.setExcluded(entry.getPositions().stream().anyMatch(position -> fileStatus.getExcludedPages().contains(position.getPageNumber())) //
&& entry.getManualChanges().stream().noneMatch(m -> m.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY))));
} }
Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId); Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -217,7 +217,6 @@ public class FileStatusService {
.dossierTemplateId(dossier.getDossierTemplateId()) .dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed()) .lastProcessed(fileModel.getLastProcessed())
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build(); .build();
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -71,7 +71,7 @@ public class DownloadPreparationService {
downloadStatus.getFiles().forEach(fileEntity -> { downloadStatus.getFiles().forEach(fileEntity -> {
RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).includeUnprocessed(reportResultMessage.getIncludeUnprocessed()).build(); RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build();
log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId()); log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId());
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message); rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_QUEUE, message);
}); });

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java
View File

@ -4,12 +4,9 @@ import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import jakarta.transaction.Transactional;
import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -21,6 +18,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage; import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@Service @Service
@ -52,7 +50,6 @@ public class DownloadProcessorService {
.downloadId(downloadJob.getStorageId()) .downloadId(downloadJob.getStorageId())
.dossierId(dossier.getId()) .dossierId(dossier.getId())
.dossierTemplateId(dossierService.getDossierById(dossier.getId()).getDossierTemplateId()) .dossierTemplateId(dossierService.getDossierById(dossier.getId()).getDossierTemplateId())
.includeUnprocessed(downloadJob.getIncludeUnprocessed())
.fileIds(filenameSortedFileIds) .fileIds(filenameSortedFileIds)
.templateIds(downloadStatus.getReports().stream().map(ReportTemplateEntity::getTemplateId).collect(Collectors.toSet())) .templateIds(downloadStatus.getReports().stream().map(ReportTemplateEntity::getTemplateId).collect(Collectors.toSet()))
.build(), 1); .build(), 1);

43
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -2,19 +2,23 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import com.google.common.hash.HashFunction; import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService; import com.iqser.red.service.persistence.management.v1.processor.service.AnalysisFlagsCalculationService;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
@ -35,6 +39,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -59,17 +66,18 @@ public class ManualRedactionService {
ManualRedactionProviderService manualRedactionProviderService; ManualRedactionProviderService manualRedactionProviderService;
AnalysisFlagsCalculationService analysisFlagsCalculationService; AnalysisFlagsCalculationService analysisFlagsCalculationService;
EntityLogService entityLogService; EntityLogService entityLogService;
DictionaryManagementService dictionaryManagementService;
HashFunction hashFunction = Hashing.murmur3_128(); HashFunction hashFunction = Hashing.murmur3_128();
ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler; ManualRedactionDictionaryUpdateHandler manualRedactionDictionaryUpdateHandler;
EntityLogMergeService entityLogMergeService;
@Transactional @Transactional
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) { public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
var response = new ArrayList<ManualAddResponse>(); var response = new ArrayList<ManualAddResponse>();
Set<ManualRedactionEntryEntity> manualRedactionEntryEntities = new HashSet<>();
dossierPersistenceService.getAndValidateDossier(dossierId); var dossierEntity = dossierPersistenceService.getAndValidateDossier(dossierId);
for (AddRedactionRequest addRedactionRequest : addRedactionRequests) { for (AddRedactionRequest addRedactionRequest : addRedactionRequests) {
manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue()); manualRedactionDictionaryUpdateHandler.validateDictionariesForAdd(addRedactionRequest, addRedactionRequest.getValue());
@ -77,7 +85,7 @@ public class ManualRedactionService {
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); manualRedactionEntryEntities.add(addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest));
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId, Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
addRedactionRequest.getValue(), addRedactionRequest.getValue(),
@ -94,6 +102,17 @@ public class ManualRedactionService {
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
} }
manualRedactionEntryEntities = manualRedactionEntryEntities.stream()
.filter(manualRedactionEntry -> !manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
.collect(Collectors.toSet());
if (!manualRedactionEntryEntities.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(MagicConverter.convert(manualRedactionEntryEntities, ManualRedactionEntry.class))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierEntity, fileStatusService.getStatus(fileId), manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId); analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
reprocess(dossierId, fileId); reprocess(dossierId, fileId);
@ -233,12 +252,13 @@ public class ManualRedactionService {
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) { public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) {
List<ManualAddResponse> response = new ArrayList<>(); List<ManualAddResponse> response = new ArrayList<>();
Set<ManualResizeRedactionEntity> manualResizeRedactionEntities = new HashSet<>();
EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId); EntityLog entityLog = entityLogService.getEntityLog(dossierId, fileId);
for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) { for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest); var resizeRedaction = resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
manualResizeRedactionEntities.add(resizeRedaction);
if (resizeRedactionRequest.getComment() != null) { if (resizeRedactionRequest.getComment() != null) {
Long commentId = commentService.addCommentAndGetId(fileId, Long commentId = commentService.addCommentAndGetId(fileId,
@ -259,6 +279,19 @@ public class ManualRedactionService {
typeIdsOfModifiedDictionaries); typeIdsOfModifiedDictionaries);
} }
manualResizeRedactionEntities = manualResizeRedactionEntities.stream()
.filter(manualRedactionEntry -> !manualRedactionEntry.getUpdateDictionary())
.collect(Collectors.toSet());
if (!manualResizeRedactionEntities.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder()
.resizeRedactions(MagicConverter.convert(manualResizeRedactionEntities, ManualResizeRedaction.class))
.build();
entityLogMergeService.sendToAnalyseQueue(fileId, dossierPersistenceService.getAndValidateDossier(dossierId), fileStatusService.getStatus(fileId), manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
reprocess(dossierId, fileId); reprocess(dossierId, fileId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId); analysisFlagsCalculationService.calculateFlags(dossierId, fileId);

13
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/CvAnalysisMessageReceiver.java
View File

@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -26,12 +27,14 @@ public class CvAnalysisMessageReceiver {
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final FileStatusService fileStatusService; private final FileStatusService fileStatusService;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows @SneakyThrows
@RabbitListener(queues = MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE) @RabbitListener(queues = MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE)
public void receive(CvAnalysisServiceResponse response) { public void receive(CvAnalysisServiceResponse response) {
addFileIdToTrace(response.getFileId());
fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false); fileStatusService.setStatusAnalyse(response.getDossierId(), response.getFileId(), false);
log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE, response.getDossierId(), response.getFileId()); log.info("Received message in {} for dossierId {} and fileId {}", MessagingConfiguration.CV_ANALYSIS_RESPONSE_QUEUE, response.getDossierId(), response.getFileId());
@ -43,7 +46,7 @@ public class CvAnalysisMessageReceiver {
public void handleDLQMessage(Message failedMessage) { public void handleDLQMessage(Message failedMessage) {
var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class); var response = objectMapper.readValue(failedMessage.getBody(), CvAnalysisServiceResponse.class);
addFileIdToTrace(response.getFileId());
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId()); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, response.getDossierId(), response.getFileId());
fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1", OffsetDateTime.now().truncatedTo( fileStatusProcessingUpdateService.analysisFailed(response.getDossierId(), response.getFileId(), new FileErrorInfo("cv analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "cv-analysis-service-v1", OffsetDateTime.now().truncatedTo(
@ -51,4 +54,12 @@ public class CvAnalysisMessageReceiver {
} }
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
} }

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java
View File

@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.settings.FileMa
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -31,6 +32,7 @@ public class ImageMessageReceiver {
private final FileStatusService fileStatusService; private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows @SneakyThrows
@ -41,6 +43,7 @@ public class ImageMessageReceiver {
String dossierId = imageResponse.path("dossierId").asText(); String dossierId = imageResponse.path("dossierId").asText();
String fileId = imageResponse.path("fileId").asText(); String fileId = imageResponse.path("fileId").asText();
addFileIdToTrace(fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false); fileStatusService.setStatusAnalyse(dossierId, fileId, false);
@ -55,10 +58,19 @@ public class ImageMessageReceiver {
}); });
String dossierId = (String) imageResponse.get("dossierId"); String dossierId = (String) imageResponse.get("dossierId");
String fileId = (String) imageResponse.get("fileId"); String fileId = (String) imageResponse.get("fileId");
addFileIdToTrace(fileId);
log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId);
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, new FileErrorInfo("image analysis failed", MessagingConfiguration.IMAGE_SERVICE_DLQ, "image-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
} }
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
} }

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java
View File

@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -28,6 +29,7 @@ public class NerMessageReceiver {
private final FileStatusService fileStatusService; private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final ObservationRegistry observationRegistry;
@SneakyThrows @SneakyThrows
@ -39,6 +41,7 @@ public class NerMessageReceiver {
String dossierId = (String) entityResponse.get("dossierId"); String dossierId = (String) entityResponse.get("dossierId");
String fileId = (String) entityResponse.get("fileId"); String fileId = (String) entityResponse.get("fileId");
addFileIdToTrace(fileId);
log.info("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE, dossierId, fileId); log.info("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_RESPONSE_QUEUE, dossierId, fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false); fileStatusService.setStatusAnalyse(dossierId, fileId, false);
@ -52,10 +55,19 @@ public class NerMessageReceiver {
}); });
String dossierId = (String) entityResponse.get("dossierId"); String dossierId = (String) entityResponse.get("dossierId");
String fileId = (String) entityResponse.get("fileId"); String fileId = (String) entityResponse.get("fileId");
addFileIdToTrace(fileId);
log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId); log.warn("Received message {} for dossierId {} and fileId {}", MessagingConfiguration.NER_SERVICE_DLQ, dossierId, fileId);
fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId,
new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); new FileErrorInfo("ner service failed", MessagingConfiguration.NER_SERVICE_DLQ, "entity-recognition-service-v3", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
} }
// This can be removed if tracing is implemented in python services.
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
} }

8
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -54,6 +54,7 @@ import io.micrometer.common.KeyValues;
import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.aop.TimedAspect;
import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.observation.ObservationRegistry; import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.aop.ObservedAspect;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@ -78,6 +79,13 @@ public class Application implements ApplicationContextAware {
} }
@Bean
public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
return new ObservedAspect(observationRegistry);
}
@Bean @Bean
public TimedAspect timedAspect(MeterRegistry registry) { public TimedAspect timedAspect(MeterRegistry registry) {

19
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
View File

@ -4,22 +4,19 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.mock.web.MockMultipartFile; import org.springframework.mock.web.MockMultipartFile;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient; import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
@ -31,6 +28,8 @@ import com.iqser.red.service.peristence.v1.server.integration.service.FileTester
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver; import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadReportMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver; import com.iqser.red.service.persistence.management.v1.processor.service.download.RedactionResultMessageReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DownloadReadyJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
@ -45,6 +44,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ -93,8 +93,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
DownloadStatusPersistenceService downloadStatusPersistenceService; DownloadStatusPersistenceService downloadStatusPersistenceService;
@BeforeEach @BeforeEach
public void createTestData() { public void createTestData() {
@ -140,7 +138,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
ReportResultMessage reportResultMessage = new ReportResultMessage(); ReportResultMessage reportResultMessage = new ReportResultMessage();
reportResultMessage.setUserId(USER_ID); reportResultMessage.setUserId(USER_ID);
reportResultMessage.setDownloadId(downloadId); reportResultMessage.setDownloadId(downloadId);
reportResultMessage.setIncludeUnprocessed(true);
downloadReportMessageReceiver.receive(reportResultMessage); downloadReportMessageReceiver.receive(reportResultMessage);
redactionResultMessageReceiver.receive(RedactionResultMessage.builder() redactionResultMessageReceiver.receive(RedactionResultMessage.builder()

7
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java
View File

@ -133,12 +133,7 @@ public class EntityLogMergeTest {
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder() EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder()
.dossierTemplateId(dossierTemplateId) .dossierTemplateId(dossierTemplateId)
.build(), FileModel.builder().id(fileId).build()); .build());
Mockito.verify(rabbitTemplate).convertAndSend(eq(MessagingConfiguration.REDACTION_PRIORITY_QUEUE), captor.capture());
AnalyzeRequest analyzeRequest = captor.getValue();
assertEquals(analyzeRequest.getManualRedactions().getEntriesToAdd().size(), 1);
assertEquals(analyzeRequest.getManualRedactions().getResizeRedactions().size(), 1);
assertNotNull(response); assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty()); assertFalse(response.getEntityLogEntry().isEmpty());

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeRequest.java
View File

@ -28,8 +28,6 @@ public class AnalyzeRequest {
private OffsetDateTime lastProcessed; private OffsetDateTime lastProcessed;
private int analysisNumber; private int analysisNumber;
@Builder.Default
private Set<Integer> excludedPages = new HashSet<>();
@Builder.Default @Builder.Default
private Set<Integer> sectionsToReanalyse = new HashSet<>(); private Set<Integer> sectionsToReanalyse = new HashSet<>();