Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7317: Endpoint to change entity types of dict-based annotations

Browse Source 
* major refactor of manual redaction service
This commit is contained in:
Kilian Schuettler 2023-08-29 18:07:42 +02:00
parent d1d5331202
commit aab2971d6e
23 changed files with 357 additions and 771 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java vendored
View File

@ -32,7 +32,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comments;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
@ -88,7 +90,7 @@ public class ManualRedactionController implements ManualRedactionResource {
.message("Comment was removed.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
manualRedactionService.deleteComment(fileId, List.of(Long.valueOf(commentId)));
manualRedactionService.deleteComment(dossierId, fileId, List.of(Long.valueOf(commentId)));
}
@ -100,6 +102,7 @@ public class ManualRedactionController implements ManualRedactionResource {
return manualRedactionService.getManualRedactions(fileId);
}
@Override
@PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
public Comments getComments(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
@ -118,7 +121,8 @@ public class ManualRedactionController implements ManualRedactionResource {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewerOrApprover(dossierId, fileId);
var response = manualRedactionService.addComment(fileId,
var response = manualRedactionService.addComment(dossierId,
fileId,
annotationId,
CommentRequest.builder().user(KeycloakSecurity.getUserId()).text(addCommentRequest.getText()).build());
@ -298,15 +302,17 @@ public class ManualRedactionController implements ManualRedactionResource {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsMemberOrApprover(dossierId);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest> requests = recategorizationRequests.stream()
.map(recategorizationRequest -> com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest.builder()
List<RecategorizationRequest> requests = recategorizationRequests.stream()
.map(recategorizationRequest -> RecategorizationRequest.builder()
.annotationId(recategorizationRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
.status(AnnotationStatus.APPROVED)
.typeId(toTypeId(recategorizationRequest.getType(), dossier.getDossierTemplateId()))
.dossierTemplateTypeId(toTypeId(recategorizationRequest.getType(), dossier.getDossierTemplateId()))
.comment(recategorizationRequest.getComment())
.dossierId(dossierId)
.addToDictionary(recategorizationRequest.isAddToDictionary())
.addToAllDossiers(recategorizationRequest.isAddToAllDossiers())
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.build())
.collect(Collectors.toList());

2
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/RedactionLogController.java vendored
View File

@ -52,7 +52,7 @@ public class RedactionLogController implements RedactionLogResource {
@RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives) {
try {
return redactionLogService.getRedactionLog(dossierId, fileId, excludedTypes, withManualRedactions, includeFalsePositives);
return redactionLogService.getRedactionLog(dossierId, fileId, excludedTypes);
} catch (FeignException e) {
throw processFeignException(e);
}

2
persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/RedactionLogInternalController.java
View File

@ -25,7 +25,7 @@ public class RedactionLogInternalController implements RedactionLogResource {
@RequestParam(value = "withManualRedactions", required = false, defaultValue = "true") boolean withManualRedactions,
@RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives) {
return redactionLogService.getRedactionLog(dossierId, fileId, excludedTypes, withManualRedactions, includeFalsePositives);
return redactionLogService.getRedactionLog(dossierId, fileId, excludedTypes);
}
}

28
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java
View File

@ -1,18 +1,24 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.annotations;
import java.time.OffsetDateTime;
import java.util.HashSet;
import java.util.Set;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import jakarta.persistence.CollectionTable;
import jakarta.persistence.Column;
import jakarta.persistence.ElementCollection;
import jakarta.persistence.EmbeddedId;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -43,8 +49,22 @@ public class ManualImageRecategorizationEntity implements IBaseAnnotation {
private OffsetDateTime softDeletedTime;
@Column
private int page;
@Column
private boolean addToDictionary;
@Column
private boolean addToAllDossiers;
@ManyToOne
private FileEntity fileStatus;
@ElementCollection
@CollectionTable(name = "manual_recategorization_type_ids_of_dictionaries_with_add")
@Fetch(value = FetchMode.SUBSELECT)
private Set<String> typeIdsOfDictionariesWithAdd = new HashSet<>();
@ElementCollection
@CollectionTable(name = "manual_recategorization_type_ids_of_dictionaries_with_delete")
@Fetch(value = FetchMode.SUBSELECT)
private Set<String> typeIdsOfDictionariesWithDelete = new HashSet<>();
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java
View File

@ -36,7 +36,7 @@ public class AnalysisFlagsCalculationService {
long startTime = System.currentTimeMillis();
var file = fileStatusPersistenceService.getStatus(fileId);
var redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true, false);
var redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee());

14
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -9,12 +9,14 @@ import java.nio.file.StandardOpenOption;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -50,6 +52,7 @@ public class FileManagementStorageService {
return storedObjectBytes;
}
@SneakyThrows
public InputStream getObject(String tenantId, String storageId) {
@ -83,11 +86,20 @@ public class FileManagementStorageService {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("RedactionLog does not exist");
throw new NotFoundException("RedactionLog does not exist");
throw new NotFoundException(String.format("RedactionLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
} catch (StorageException e) {
throw new InternalServerErrorException(e.getMessage());
}
}
public void storeRedactionLog(String dossierId, String fileId, RedactionLog redactionLog) {
storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), redactionLog);
}
public SectionGrid getSectionGrid(String dossierId, String fileId) {
try {

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java
View File

@ -1,7 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import jakarta.transaction.Transactional;
import org.apache.commons.lang3.StringUtils;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.bind.annotation.RestController;
@ -17,6 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.search.v1.model.IndexMessageType;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -54,7 +53,7 @@ public class FileStatusProcessingUpdateService {
indexingService.addToIndexingQueue(IndexMessageType.INSERT, dossier.getDossierTemplateId(), dossierId, fileId, 2);
}
manualRedactionService.updateProcessedDate(fileId, analyzeResult.getManualRedactions());
manualRedactionService.updateProcessedDate(analyzeResult.getManualRedactions());
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (analyzeResult.getAddedFileAttributes() != null && !analyzeResult.getAddedFileAttributes().isEmpty()) {

59
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/RedactionLogService.java
View File

@ -2,26 +2,17 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.FilteredRedactionLogRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
@Service
@ -29,59 +20,29 @@ import lombok.RequiredArgsConstructor;
public class RedactionLogService {
private final FileManagementStorageService fileManagementStorageService;
private final ManualRedactionProviderService manualRedactionService;
private final DossierPersistenceService dossierPersistenceService;
private final FileStatusService fileStatusService;
private final ColorsService colorsService;
private final DictionaryPersistenceService dictionaryPersistenceService;
private final RedactionLogMergeService redactionLogMergeService;
public RedactionLog getRedactionLog(String dossierId, String fileId, boolean withManualRedactions, boolean includeFalsePositives) {
public RedactionLog getRedactionLog(String dossierId, String fileId) {
return getRedactionLog(dossierId, fileId, null, withManualRedactions, includeFalsePositives);
return getRedactionLog(dossierId, fileId, Collections.emptyList());
}
public RedactionLog getRedactionLog(String dossierId, String fileId, List<String> excludedTypes, boolean withManualRedactions, boolean includeFalsePositives) {
public RedactionLog getRedactionLog(String dossierId, String fileId, List<String> excludedTypes) {
var fileStatus = fileStatusService.getStatus(fileId);
RedactionLog redactionLog;
if (withManualRedactions) {
var dossier = dossierPersistenceService.findByDossierId(dossierId);
var manualRedactions = manualRedactionService.getManualRedactions(fileId);
var colors = MagicConverter.convert(colorsService.getColors(dossier.getDossierTemplateId()), Colors.class);
var types = MagicConverter.convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossier.getDossierTemplateId(), true), Type.class);
var dossierTypes = MagicConverter.convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId, true), Type.class);
types.addAll(dossierTypes);
try {
redactionLog = redactionLogMergeService.provideRedactionLog(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.manualRedactions(manualRedactions)
.dossierTemplateId(dossier.getDossierTemplateId())
.excludedPages(fileStatus.getExcludedPages())
.includeFalsePositives(includeFalsePositives)
.colors(colors)
.types(types)
.build());
} catch (FeignException e) {
if (e.status() == HttpStatus.NOT_FOUND.value()) {
throw new NotFoundException(e.getMessage());
}
throw e;
}
} else {
redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId);
}
redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId);
if (fileStatus.isExcluded()) {
redactionLog.setRedactionLogEntry(new ArrayList<>());
}
if (fileStatus.isHasUpdates())
if (excludedTypes != null) {
redactionLog.getRedactionLogEntry().removeIf(nextEntry -> excludedTypes.contains(nextEntry.getType()));
}
@ -102,11 +63,7 @@ public class RedactionLogService {
filteredRedactionLogRequest.setSpecifiedDate(OffsetDateTime.MIN);
}
var redactionLog = getRedactionLog(dossierId,
fileId,
filteredRedactionLogRequest.getExcludedTypes(),
filteredRedactionLogRequest.isWithManualRedactions(),
filteredRedactionLogRequest.isIncludeFalsePositives());
var redactionLog = getRedactionLog(dossierId, fileId, filteredRedactionLogRequest.getExcludedTypes());
var redactionLogEntries = redactionLog.getRedactionLogEntry();
Iterator<RedactionLogEntry> it = redactionLogEntries.iterator();

109
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionDictionaryUpdateHandler.java
View File

@ -1,19 +1,26 @@
package com.iqser.red.service.persistence.management.v1.processor.service.manualredactions;
import java.util.Collections;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithDictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithAddToDictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithRemoveFromDictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
@ -25,33 +32,46 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionDictionaryUpdateHandler {
private final DictionaryManagementService dictionaryManagementService;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final FileStatusPersistenceService fileStatusPersistenceService;
private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
private final DossierPersistenceService dossierPersistenceService;
public Set<String> handleAddToDictionary(String fileId, String dossierId, ManualRequestWithDictionary manualRequestWithDictionary) {
public Set<String> handleAddToDictionaryAndReturnModifiedTypeIds(String fileId, String annotationId, ManualRequestWithAddToDictionary manualRequestWithAddToDictionary) {
if (!manualRequestWithDictionary.isApproved()) {
return Collections.emptySet();
if (!manualRequestWithAddToDictionary.isApproved()) {
return null;
}
if (!manualRequestWithAddToDictionary.isAddToDictionary()) {
addRedactionPersistenceService.updateStatus(fileId, annotationId, manualRequestWithAddToDictionary.getStatus(), false, null);
return null;
}
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
String dossierTemplateTypeId = manualRequestWithDictionary.getDossierTemplateTypeId();
String value = manualRequestWithDictionary.getValue();
DictionaryEntryType dictionaryEntryType = manualRequestWithDictionary.getDictionaryEntryType();
if (manualRequestWithDictionary.isAddToAllDossiers()) {
List<DictionaryEntry> dictionaryEntriesToUnDelete = dictionaryManagementService.getAllEntriesInDossierTemplate(dossierTemplateTypeId, value);
if (manualRequestWithAddToDictionary.isAddToAllDossiers()) {
var dictionaryEntriesToUnDelete = dictionaryManagementService.getAllEntriesInDossierTemplate(manualRequestWithAddToDictionary.getDossierTemplateTypeId(),
manualRequestWithAddToDictionary.getValue());
dictionaryEntriesToUnDelete.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
addToDictionary(entry.getTypeId(), manualRequestWithDictionary.getValue(), dossierId, fileId, manualRequestWithDictionary.getDictionaryEntryType());
addToDictionary(entry.getTypeId(), manualRequestWithAddToDictionary.getValue(), manualRequestWithAddToDictionary.getDossierId(), fileId, manualRequestWithAddToDictionary.getDictionaryEntryType());
});
addToDictionary(dossierTemplateTypeId, value, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(dossierTemplateTypeId);
} else {
addToDictionary(dossierTemplateTypeId + ":" + dossierId, value, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(dossierTemplateTypeId + ":" + dossierId);
addToDictionary(manualRequestWithAddToDictionary.getDossierTemplateTypeId(),
manualRequestWithAddToDictionary.getValue(),
manualRequestWithAddToDictionary.getDossierId(),
fileId,
manualRequestWithAddToDictionary.getDictionaryEntryType());
typeIdsOfModifiedDictionaries.add(manualRequestWithAddToDictionary.getDossierTemplateTypeId());
return typeIdsOfModifiedDictionaries;
}
addToDictionary(manualRequestWithAddToDictionary.getDossierTemplateTypeId() + ":" + manualRequestWithAddToDictionary.getDossierId(),
manualRequestWithAddToDictionary.getValue(),
manualRequestWithAddToDictionary.getDossierId(),
fileId,
manualRequestWithAddToDictionary.getDictionaryEntryType());
typeIdsOfModifiedDictionaries.add(manualRequestWithAddToDictionary.getDossierTemplateTypeId() + ":" + manualRequestWithAddToDictionary.getDossierId());
return typeIdsOfModifiedDictionaries;
}
@ -89,4 +109,53 @@ public class ManualRedactionDictionaryUpdateHandler {
}
}
public Set<String> handleRemoveFromDictionaryAndReturnModifiedTypeIds(RedactionLogEntry redactionLogEntry,
String fileId,
String dossierId,
String dossierTemplateId,
ManualRequestWithRemoveFromDictionary manualRequestWithRemoveFromDictionary) {
if (!manualRequestWithRemoveFromDictionary.isApproved()) {
return null;
}
if (!manualRequestWithRemoveFromDictionary.isRemoveFromDictionary()) {
return null;
}
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (manualRequestWithRemoveFromDictionary.isRemoveFromAllDossiers()) {
var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(), dossierTemplateId),
redactionLogEntry.getValue());
dictionaryEntriesToRemove.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
removeFromDictionary(entry.getTypeId(), entry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
});
} else {
typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId));
removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossierTemplateId, dossierId), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
}
// This is needed to remove resizeRedactions with addToDictionary.
removeResizeRedactionsWithAddToDictionary(dossierTemplateId, redactionLogEntry.getValue());
return typeIdsOfModifiedDictionaries;
}
private void removeResizeRedactionsWithAddToDictionary(String dossierTemplateId, String redactionLogEntryValue) {
var resizeRedactionsWithSameValue = resizeRedactionPersistenceService.findByAnnotationStatusAndValue(AnnotationStatus.APPROVED, redactionLogEntryValue);
resizeRedactionsWithSameValue.forEach(resizeRedaction -> {
var file = fileStatusPersistenceService.getStatus(resizeRedaction.getId().getFileId());
var dossierForResizeRedaction = dossierPersistenceService.findByDossierId(file.getDossierId());
if (!file.getWorkflowStatus().equals(WorkflowStatus.APPROVED) && dossierTemplateId.equals(dossierForResizeRedaction.getDossierTemplateId())) {
resizeRedactionPersistenceService.hardDelete(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId());
}
});
}
}

118
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -54,15 +54,15 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAddResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithDictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRequestWithAddToDictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
@ -110,7 +110,16 @@ public class ManualRedactionService {
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
manualRedactionDictionaryUpdateHandler.handleAddToDictionary(fileId, annotationId, addRedactionRequest);
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
annotationId,
addRedactionRequest);
addRedactionPersistenceService.updateStatus(fileId,
annotationId,
addRedactionRequest.getStatus(),
typeIdsOfModifiedDictionaries != null,
typeIdsOfModifiedDictionaries);
Long commentId = addCommentAndGetId(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser());
@ -127,7 +136,7 @@ public class ManualRedactionService {
}
private void validateDictionaries(ManualRequestWithDictionary addRedactionRequest) {
private void validateDictionaries(ManualRequestWithAddToDictionary addRedactionRequest) {
if (addRedactionRequest.isAddToDictionary()) {
if (addRedactionRequest.isAddToAllDossiers()) {
@ -139,7 +148,7 @@ public class ManualRedactionService {
}
private void validateDictionary(ManualRequestWithDictionary addRedactionRequest) {
private void validateDictionary(ManualRequestWithAddToDictionary addRedactionRequest) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
@ -263,11 +272,20 @@ public class ManualRedactionService {
requiresReAnalysis = requiresReAnalysis || matchingEntryFound;
}
boolean removedFromDictionary = handleRemoveFromDictionary(getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()),
dossier,
fileId,
Set<String> typeIdsOfModifiedDictionaries = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(//
getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId()),//
dossierId, //
dossier.getDossierTemplateId(),//
fileId, //
removeRedactionRequest);
boolean removedFromDictionary = typeIdsOfModifiedDictionaries != null;
removeRedactionPersistenceService.updateStatus(fileId,
removeRedactionRequest.getAnnotationId(),
removeRedactionRequest.getStatus(),
removedFromDictionary,
typeIdsOfModifiedDictionaries);
if (!matchingEntryFound && !removedFromDictionary && idRemoval.isApproved()) {
removeRedactionPersistenceService.markAsProcessed(idRemoval);
}
@ -311,40 +329,7 @@ public class ManualRedactionService {
}
private boolean handleRemoveFromDictionary(RedactionLogEntry redactionLogEntry, DossierEntity dossier, String fileId, RemoveRedactionRequest removeRedactionRequest) {
if (!removeRedactionRequest.isApproved()) {
return false;
}
if (!removeRedactionRequest.isRemoveFromDictionary()) {
removeRedactionPersistenceService.updateStatus(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), false, Collections.emptySet());
return false;
}
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (removeRedactionRequest.isRemoveFromAllDossiers()) {
var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId()),
redactionLogEntry.getValue());
dictionaryEntriesToRemove.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
removeFromDictionary(entry.getTypeId(), entry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY);
});
} else {
typeIdsOfModifiedDictionaries.add(toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()));
removeFromDictionary(toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()),
redactionLogEntry.getValue(),
dossier.getId(),
fileId,
DictionaryEntryType.ENTRY);
}
// This is needed to remove resizeRedactions with addToDictionary.
removeResizeRedactionsWithAddToDictionary(dossier.getDossierTemplateId(), redactionLogEntry.getValue());
removeRedactionPersistenceService.updateStatus(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), true, typeIdsOfModifiedDictionaries);
return true;
}
private RedactionLogEntry getRedactionLogEntry(RedactionLog redactionLog, String annotationId) {
@ -357,18 +342,6 @@ public class ManualRedactionService {
}
private void removeResizeRedactionsWithAddToDictionary(String dossierTemplateId, String redactionLogEntryValue) {
var resizeRedactionsWithSameValue = resizeRedactionPersistenceService.findByAnnotationStatusAndValue(AnnotationStatus.APPROVED, redactionLogEntryValue);
resizeRedactionsWithSameValue.forEach(resizeRedaction -> {
var file = fileStatusPersistenceService.getStatus(resizeRedaction.getId().getFileId());
var dossierForResizeRedaction = dossierPersistenceService.findByDossierId(file.getDossierId());
if (!file.getWorkflowStatus().equals(WorkflowStatus.APPROVED) && dossierTemplateId.equals(dossierForResizeRedaction.getDossierTemplateId())) {
resizeRedactionPersistenceService.hardDelete(resizeRedaction.getId().getFileId(), resizeRedaction.getId().getAnnotationId());
}
});
}
@Transactional
public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId, List<ForceRedactionRequest> forceRedactionRequests) {
@ -420,11 +393,31 @@ public class ManualRedactionService {
var response = new ArrayList<ManualAddResponse>();
var requiresReanalysis = false;
dossierPersistenceService.getAndValidateDossier(dossierId);
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId);
for (var recategorizationRequest : recategorizationRequests) {
validateDictionaries(recategorizationRequest);
recategorizationPersistenceService.insert(fileId, recategorizationRequest);
Set<String> typeIdsOfDictionariesWithAdd = manualRedactionDictionaryUpdateHandler.handleAddToDictionaryAndReturnModifiedTypeIds(fileId,
recategorizationRequest.getAnnotationId(),
recategorizationRequest);
Set<String> typeIdsOfDictionariesWithDelete = manualRedactionDictionaryUpdateHandler.handleRemoveFromDictionaryAndReturnModifiedTypeIds(//
getRedactionLogEntry(redactionLog, recategorizationRequest.getAnnotationId()),//
recategorizationRequest.getDossierId(),//
dossier.getDossierTemplateId(),//
fileId,//
recategorizationRequest);
recategorizationPersistenceService.updateStatus(fileId,
recategorizationRequest.getAnnotationId(),
AnnotationStatus.APPROVED,
typeIdsOfDictionariesWithAdd != null,
typeIdsOfDictionariesWithAdd,
typeIdsOfDictionariesWithDelete);
Long commentId = addCommentAndGetId(fileId, recategorizationRequest.getAnnotationId(), recategorizationRequest.getComment(), recategorizationRequest.getUser());
requiresReanalysis = requiresReanalysis || recategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED);
@ -441,12 +434,23 @@ public class ManualRedactionService {
}
public CommentEntity addComment(String fileId, String annotationId, CommentRequest commentRequest) {
public CommentEntity addComment(String dossierId, String fileId, String annotationId, CommentRequest commentRequest) {
CommentEntity createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
fileStatusPersistenceService.updateHasComments(fileId, true);
if (!fileStatusService.getStatus(fileId).getProcessingStatus().equals(ProcessingStatus.PROCESSED)) {
reprocess(dossierId, fileId);
return createdComment;
}
fileStatusService.setStatusProcessing(fileId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId);
redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.forEach(entry -> entry.getComments().add(MagicConverter.convert(createdComment, RedactionLogComment.class)));
fileManagementStorageService.storeRedactionLog(dossierId, fileId, redactionLog);
fileStatusService.setStatusProcessed(fileId);
return createdComment;
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -119,7 +119,7 @@ public class AddRedactionPersistenceService {
boolean isAddOrRemoveFromDictionary,
Set<String> typeIdsOfModifiedDictionaries) {
var addRedaction = manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId))
ManualRedactionEntryEntity addRedaction = manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId))
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
addRedaction.setStatus(annotationStatus);

20
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@ -43,6 +44,25 @@ public class ImageRecategorizationPersistenceService {
imageRecategorizationRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus);
}
@Transactional
public void updateStatus(String fileId,
String annotationId,
AnnotationStatus annotationStatus,
boolean modifiedDictionary,
Set<String> typeIdsOfDictionaryWithAdd,
Set<String> typeIdsOfDictionaryWithDelete) {
ManualImageRecategorizationEntity addRedaction = imageRecategorizationRepository.findById(new AnnotationEntityId(annotationId, fileId))
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
addRedaction.setStatus(annotationStatus);
addRedaction.setAddToDictionary(modifiedDictionary);
addRedaction.setTypeIdsOfDictionariesWithAdd(typeIdsOfDictionaryWithAdd);
addRedaction.setTypeIdsOfDictionariesWithDelete(typeIdsOfDictionaryWithDelete);
imageRecategorizationRepository.saveAndFlush(addRedaction);
}
@Transactional
public void hardDelete(String fileId, String annotationId) {

549
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/redactionlog/RedactionLogMergeService.java
View File

@ -1,549 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.service.redactionlog;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.utils.ColorUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import io.micrometer.core.annotation.Timed;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class RedactionLogMergeService {
private static final String DELETED_TYPE_COLOR = "#9398a0";
private final SectionTextService sectionTextService;
private final FileManagementStorageService fileManagementStorageService;
@Timed("redactmanager_getMergedRedactionLog")
public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
log.debug("Requested preview for: {}", redactionRequest);
var redactionLog = fileManagementStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
if (redactionLog == null) {
throw new NotFoundException("RedactionLog not present");
}
log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
var merged = mergeRedactionLogData(redactionLog,
redactionRequest.getManualRedactions(),
redactionRequest.getExcludedPages(),
redactionRequest.getTypes(),
redactionRequest.getColors());
merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
return merged;
}
private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, ManualRedactions manualRedactions, Set<Integer> excludedPages, List<Type> types, Colors colors) {
var skippedImportedRedactions = new HashSet<>();
log.info("Merging Redaction log with manual redactions");
if (manualRedactions != null) {
var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
var applicableManualRedactions = manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList());
if (!applicableManualRedactions.isEmpty()) {
processRedactionLogEntry(applicableManualRedactions, entry, types, colors);
}
if (entry.isImported() && !entry.isRedacted()) {
skippedImportedRedactions.add(entry.getId());
}
entry.setComments(convert(manualRedactions.getComments().get(entry.getId())));
if (excludedPages != null && !excludedPages.isEmpty()) {
entry.getPositions().forEach(pos -> {
if (!isLocalManualRedaction(entry) && excludedPages.contains(pos.getPage())) {
entry.setExcluded(true);
}
});
}
}
}
Set<String> processedIds = new HashSet<>();
redactionLog.getRedactionLogEntry().removeIf(entry -> {
if (entry.getImportedRedactionIntersections() != null) {
entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
.equals("ocr")))) {
return true;
}
}
if (processedIds.contains(entry.getId())) {
log.info("Duplicate annotation found with id {}", entry.getId());
return true;
}
processedIds.add(entry.getId());
return false;
});
return redactionLog;
}
private boolean isLocalManualRedaction(RedactionLogEntry entry) {
return entry.getManualChanges() != null && entry.getManualChanges()
.stream()
.anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY || mc.getManualRedactionType() == ManualRedactionType.RESIZE && mc.getAnnotationStatus() == AnnotationStatus.APPROVED && entry.getEngines()
.contains(Engine.RULE) && !entry.getEngines().contains(Engine.DICTIONARY));
}
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
manualRedactions.getImageRecategorization().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getIdsToRemove().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getForceRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getLegalBasisChanges().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getResizeRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
Collections.sort(manualRedactionWrappers);
return manualRedactionWrappers;
}
private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
manualRedactionWrappers.forEach(mrw -> {
Object item = mrw.getItem();
if (item instanceof ManualImageRecategorization) {
var imageRecategorization = (ManualImageRecategorization) item;
processManualRecategorization(redactionLogEntry, types, colors, imageRecategorization);
}
if (item instanceof IdRemoval) {
var manualRemoval = (IdRemoval) item;
processIdRemoval(redactionLogEntry, types, colors, manualRemoval);
}
if (item instanceof ManualForceRedaction) {
var manualForceRedact = (ManualForceRedaction) item;
processManualForceRedaction(redactionLogEntry, types, colors, manualForceRedact);
}
if (item instanceof ManualLegalBasisChange) {
var manualLegalBasisChange = (ManualLegalBasisChange) item;
processManualLegalBasisChange(redactionLogEntry, types, colors, manualLegalBasisChange);
}
if (item instanceof ManualResizeRedaction) {
var manualResizeRedact = (ManualResizeRedaction) item;
processManualResizeRedaction(redactionLogEntry, types, colors, manualResizeRedact);
}
});
}
private void processManualRecategorization(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualImageRecategorization imageRecategorization) {
String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setType(imageRecategorization.getType());
if (isHint(types, imageRecategorization.getType())) {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setHint(true);
} else {
redactionLogEntry.setHint(false);
}
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
redactionLogEntry.getManualChanges()
.add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
}
private String mergeReasonIfNecessary(String currentReason, String addition) {
if (currentReason != null) {
if (!currentReason.contains(addition)) {
return currentReason + addition;
}
return currentReason;
} else {
return "";
}
}
private void processIdRemoval(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, IdRemoval manualRemoval) {
boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
log.debug("Skipping merge for dictionary-modifying entry");
} else {
String redactionLogEntryType = redactionLogEntry.getType();
String manualOverrideReason = null;
if (isApprovedRedaction) {
redactionLogEntry.setRedacted(false);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntryType, colors, false, redactionLogEntry.isRedacted(), true, types));
redactionLogEntry.setHint(false);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setColor(getColor(redactionLogEntryType, colors, true, redactionLogEntry.isRedacted(), false, types));
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
}
redactionLogEntry.getManualChanges()
.add(ManualChange.from(manualRemoval)
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
}
private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
}
private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualForceRedaction manualForceRedact) {
String manualOverrideReason = null;
var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
// Forcing a skipped hint should result in a hint
if (dictionaryIsHint) {
redactionLogEntry.setHint(true);
} else {
redactionLogEntry.setRedacted(true);
}
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
redactionLogEntry.getManualChanges().add(manualChange);
}
private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualLegalBasisChange manualLegalBasisChange) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
redactionLogEntry.setRedacted(true);
if (manualLegalBasisChange.getSection() != null) {
redactionLogEntry.setSection(manualLegalBasisChange.getSection());
}
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
redactionLogEntry.setValue(manualLegalBasisChange.getValue());
}
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
if (manualLegalBasisChange.getSection() != null) {
manualChange.withChange("section", manualLegalBasisChange.getSection());
}
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
manualChange.withChange("value", manualLegalBasisChange.getValue());
}
redactionLogEntry.getManualChanges().add(manualChange);
}
private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualResizeRedaction manualResizeRedact) {
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
redactionLogEntry.setValue(manualResizeRedact.getValue());
}
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
}
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
}
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.getManualChanges()
.add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
}
public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments,
Colors colors,
List<Type> types,
int analysisNumber) {
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (shouldCreateManualEntry(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
redactionLogEntries.add(redactionLogEntry);
}
}
return redactionLogEntries;
}
private List<RedactionLogComment> convert(List<Comment> comments) {
return comments == null ? null : comments.stream()
.map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
.collect(Collectors.toList());
}
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle> positions) {
return positions.stream()
.map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
.collect(Collectors.toList());
}
@SuppressWarnings("PMD.UselessParentheses")
private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, Colors colors, List<Type> types, int analysisNumber) {
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
List<ManualChange> changeList = new ArrayList<>();
changeList.add(change);
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.sourceId(manualRedactionEntry.getSourceId())
.section(manualRedactionEntry.getSection())
.type(manualRedactionEntry.getType())
.redacted(true)
.isHint(false)
.sectionNumber(-1)
.rectangle(manualRedactionEntry.isRectangle())
.manualChanges(changeList)
.changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
.build();
}
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, List<Type> types) {
if (requested) {
return ColorUtils.convertColor(colors.getRequestRemoveColor());
}
if (skipped || !isRedaction && !isHint(types, type)) {
return ColorUtils.convertColor(colors.getSkippedColor());
}
return getColor(types, type);
}
private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
if (status.equals(AnnotationStatus.REQUESTED)) {
return ColorUtils.convertColor(colors.getRequestAddColor());
} else if (status.equals(AnnotationStatus.DECLINED)) {
return ColorUtils.convertColor(colors.getSkippedColor());
}
return getColor(types, type);
}
private float[] getColor(List<Type> types, String type) {
var matchingTypes = getMatchingTypes(types, type);
Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
if (foundAndNotDeletedType.isPresent()) {
return ColorUtils.convertColor(foundAndNotDeletedType.get().getHexColor());
}
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
return firstDeletedType.map(value -> ColorUtils.convertColor(value.getHexColor())).orElseGet(() -> ColorUtils.convertColor(DELETED_TYPE_COLOR));
}
boolean isHint(List<Type> types, String type) {
var matchingTypes = getMatchingTypes(types, type);
Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
if (foundAndNotDeletedType.isPresent()) {
return foundAndNotDeletedType.get().isHint();
}
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
return firstDeletedType.map(Type::isHint).orElse(false);
}
private List<Type> getMatchingTypes(List<Type> types, String type) {
return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
}
private boolean isDeletedType(Type type) {
return type.getSoftDeletedTime() != null;
}
@Data
@AllArgsConstructor
private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
private String id;
private OffsetDateTime date;
private Object item;
@Override
public int compareTo(ManualRedactionWrapper o) {
return this.date.compareTo(o.date);
}
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -155,4 +155,6 @@ databaseChangeLog:
file: db/changelog/tenant/106-add-add-to-all-dossiers-to-resize-redactions.yaml
- include:
file: db/changelog/tenant/107-add-last-layout-processed-column.yaml
- include:
file: db/changelog/tenant/108-added-dictionary-changes-to-manual-recategorization.yaml

79
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/108-added-dictionary-changes-to-manual-recategorization.yaml Normal file
View File

@ -0,0 +1,79 @@
databaseChangeLog:
- changeSet:
id: added-dictionary-changes-to-manual-recategorization
author: kilian
changes:
- addColumn:
columns:
- column:
name: add_to_dictionary
type: BOOLEAN
tableName: manual_image_recategorization
- addColumn:
columns:
- column:
name: add_to_all_dossiers
type: BOOLEAN
tableName: manual_image_recategorization
- changeSet:
id: added-dictionary-changes-to-manual-recategorization-2
author: kilian
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
name: manual_image_recategorization_entity_annotation_id
type: VARCHAR(255)
- column:
constraints:
nullable: false
name: manual_image_recategorization_entity_file_id
type: VARCHAR(255)
- column:
name: type_ids_of_dictionaries_with_add
type: VARCHAR(255)
tableName: manual_recategorization_type_ids_of_dictionaries_with_add
- createTable:
columns:
- column:
constraints:
nullable: false
name: manual_image_recategorization_entity_annotation_id
type: VARCHAR(255)
- column:
constraints:
nullable: false
name: manual_image_recategorization_entity_file_id
type: VARCHAR(255)
- column:
name: type_ids_of_dictionaries_with_delete
type: VARCHAR(255)
tableName: manual_recategorization_type_ids_of_dictionaries_with_delete
- changeSet:
id: added-dictionary-changes-to-manual-recategorization-3
author: kilian
changes:
- addForeignKeyConstraint:
baseColumnNames: manual_image_recategorization_entity_annotation_id, manual_image_recategorization_entity_file_id
baseTableName: manual_recategorization_type_ids_of_dictionaries_with_add
constraintName: fk_manual_recategorization_to_dictionary_add
deferrable: false
initiallyDeferred: false
onDelete: NO ACTION
onUpdate: NO ACTION
referencedColumnNames: annotation_id, file_id
referencedTableName: manual_image_recategorization
validate: true
- addForeignKeyConstraint:
baseColumnNames: manual_image_recategorization_entity_annotation_id, manual_image_recategorization_entity_file_id
baseTableName: manual_recategorization_type_ids_of_dictionaries_with_delete
constraintName: fk_manual_recategorization_to_dictionary_delete
deferrable: false
initiallyDeferred: false
onDelete: NO ACTION
onUpdate: NO ACTION
referencedColumnNames: annotation_id, file_id
referencedTableName: manual_image_recategorization
validate: true

32
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -105,7 +105,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog);
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog);
Assertions.assertThrows(FeignException.Forbidden.class,
() -> manualRedactionClient.removeRedactionBulk(dossier.getId(),
@ -254,7 +254,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog);
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog);
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
@ -302,7 +302,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog);
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog);
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
@ -343,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, redactionLog);
when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(redactionLog);
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(redactionLog);
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
@ -427,8 +427,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.REDACTION_LOG, redactionLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest1)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId(), true, true)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId())).thenReturn(redactionLog1);
var redactionLog2 = new RedactionLog(1,
1,
@ -445,8 +444,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.REDACTION_LOG, redactionLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest2)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId(), true, true)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId())).thenReturn(redactionLog2);
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
@ -576,8 +574,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.REDACTION_LOG, redactionLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest1)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId(), true, true)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId())).thenReturn(redactionLog1);
var redactionLog2 = new RedactionLog(1,
1,
@ -594,8 +591,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.REDACTION_LOG, redactionLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest2)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId(), true, true)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId())).thenReturn(redactionLog2);
// resize redaction in dossier 1
var resizeRedactionDosAndAddToAllDos = ResizeRedactionRequestModel.builder()
@ -727,8 +723,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.REDACTION_LOG, redactionLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest1)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId(), true, true)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId())).thenReturn(redactionLog1);
var redactionLog2 = new RedactionLog(1,
1,
@ -745,8 +740,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.REDACTION_LOG, redactionLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest2)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId(), true, true)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId())).thenReturn(redactionLog2);
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()
@ -875,8 +869,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.REDACTION_LOG, redactionLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest1)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId(), true, true)).thenReturn(redactionLog1);
when(redactionLogService.getRedactionLog(file1.getDossierId(), file1.getFileId())).thenReturn(redactionLog1);
var redactionLog2 = new RedactionLog(1,
1,
@ -893,8 +886,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.REDACTION_LOG, redactionLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(redactionLogMergeService.provideRedactionLog(redactionRequest2)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId(), true, true)).thenReturn(redactionLog2);
when(redactionLogService.getRedactionLog(file2.getDossierId(), file2.getFileId())).thenReturn(redactionLog2);
// resize redaction in dossier dict
var resizeRedactionDosTemp = ResizeRedactionRequestModel.builder()

47
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/RedactionLogTest.java
View File

@ -1,47 +0,0 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.time.OffsetDateTime;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.client.RedactionLogClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.FilteredRedactionLogRequest;
public class RedactionLogTest extends AbstractPersistenceServerServiceTest {
@Autowired
private FileTesterAndProvider fileTesterAndProvider;
@Autowired
private DossierTesterAndProvider dossierTesterAndProvider;
@Autowired
private RedactionLogClient redactionLogClient;
@Test
public void testRedactionLog() {
var dossier = dossierTesterAndProvider.provideTestDossier();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
assertThat(redactionLogClient.getSectionGrid(dossier.getId(), file.getId())).isNotNull();
assertThat(redactionLogClient.getRedactionLog(dossier.getId(), file.getId(), null, true, false)).isNotNull();
assertThat(redactionLogClient.getRedactionLog(dossier.getId(), file.getId(), null, false, false)).isNotNull();
assertThat(redactionLogClient.getFilteredRedactionLog(dossier.getId(),
file.getId(),
FilteredRedactionLogRequest.builder()
.excludedTypes(null)
.withManualRedactions(true)
.includeFalsePositives(false)
.specifiedDate(OffsetDateTime.now().minusDays(30))
.build())).isNotNull();
}
}

6
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -57,6 +57,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.RedactionLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
@ -87,7 +88,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -123,7 +123,7 @@ public abstract class AbstractPersistenceServerServiceTest {
@MockBean
protected SearchClient searchClient;
@MockBean
protected RedactionLogMergeService redactionLogMergeService;
protected RedactionLogService redactionLogService;
@MockBean
protected PDFTronClient pdfTronRedactionClient;
@Autowired
@ -277,7 +277,7 @@ public abstract class AbstractPersistenceServerServiceTest {
// doNothing().when(pdfTronRedactionClient).testDigitalCurrentSignature(Mockito.any());
when(amqpAdmin.getQueueInfo(Mockito.any())).thenReturn(null);
when(redactionLogMergeService.provideRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, 1, Lists.newArrayList(), null, 0, 0, 0, 0));
when(redactionLogService.getRedactionLog(Mockito.any(), Mockito.any())).thenReturn(new RedactionLog(1, 1, Lists.newArrayList(), null, 0, 0, 0, 0));
when(redactionClient.testRules(Mockito.anyString())).thenReturn(DroolsSyntaxValidation.builder().compiled(true).droolsSyntaxErrorMessages(Collections.emptyList()).build());
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/AddRedactionRequest.java
View File

@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class AddRedactionRequest implements ManualRequestWithDictionary {
public class AddRedactionRequest implements ManualRequestWithAddToDictionary {
private String user;
private String dossierTemplateTypeId;

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithDictionary.java → persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithAddToDictionary.java
View File

@ -2,9 +2,10 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.annotation
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
public interface ManualRequestWithDictionary extends BaseManualRequest{
public interface ManualRequestWithAddToDictionary extends BaseManualRequest {
String getValue();
String getDossierId();
boolean isAddToDictionary();
boolean isAddToAllDossiers();
boolean isForceAddToDictionary();

13
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualRequestWithRemoveFromDictionary.java Normal file
View File

@ -0,0 +1,13 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.annotations;
public interface ManualRequestWithRemoveFromDictionary extends BaseManualRequest {
String getAnnotationId();
boolean isRemoveFromDictionary();
boolean isRemoveFromAllDossiers();
}

16
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RecategorizationRequest.java
View File

@ -14,12 +14,13 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class RecategorizationRequest implements ManualRequestWithDictionary {
public class RecategorizationRequest implements ManualRequestWithAddToDictionary, ManualRequestWithRemoveFromDictionary {
String annotationId;
String user;
AnnotationStatus status;
String typeId;
String dossierTemplateTypeId;
String dossierId;
String comment;
int page;
String value;
@ -36,9 +37,16 @@ public class RecategorizationRequest implements ManualRequestWithDictionary {
@Override
public String getDossierTemplateTypeId() {
public boolean isRemoveFromDictionary() {
return typeId;
return addToDictionary;
}
@Override
public boolean isRemoveFromAllDossiers() {
return addToAllDossiers;
}
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/RemoveRedactionRequest.java
View File

@ -9,7 +9,7 @@ import lombok.NoArgsConstructor;
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class RemoveRedactionRequest implements BaseManualRequest{
public class RemoveRedactionRequest implements ManualRequestWithRemoveFromDictionary {
private String annotationId;
private String user;