Compare commits

...

18 Commits

Author SHA1 Message Date
Ali Oezyetimoglu
07c6ae19cf Merge branch 'RED-7400-b' into 'release/2.123.x'
RED-7400: changed place of isLocalManualRedaction as it is not needed in RedactionLogEntry anymore

See merge request redactmanager/persistence-service!81
2023-08-17 12:24:46 +02:00
Ali Oezyetimoglu
cf82406e69 RED-7400: changed place of isLocalManualRedaction as it is not needed in RedactionLogEntry anymore 2023-08-17 12:07:53 +02:00
Timo Bejan
f06f73c55c RED-7158-2 last layout processed time 2023-08-16 14:31:23 +03:00
Timo Bejan
8b6353157c RED-7158 layout status in UI 2023-08-16 14:30:46 +03:00
Kilian Schüttler
c6b3bf682f Merge branch 'DM-357-release' into 'release/2.123.x'
DM-357: always schedule reanalysis, when any manual redaction happens

See merge request redactmanager/persistence-service!72
2023-08-16 11:45:03 +02:00
Kilian Schuettler
f2f4564590 DM-357: remove outdated status and messageTypes
* remove surrounding-text functionality
2023-08-16 11:37:19 +02:00
Kilian Schuettler
49cd622e45 DM-357: schedule reanalysis for every manualRedaction
* silence that stupid automaticAnalysisJob spamming
2023-08-16 10:43:07 +02:00
Kilian Schuettler
a9c123f5b7 DM-357: always schedule reanalysis, when any manual redaction happens
* refactored code slightly
2023-08-15 20:24:53 +02:00
Kilian Schüttler
d8ac1a04db Merge branch 'RED-7158-release' into 'release/2.123.x'
RED-7158: custom release

See merge request redactmanager/persistence-service!70
2023-08-15 12:31:29 +02:00
Kilian Schuettler
44a066e204 RED-7158: custom release 2023-08-15 12:20:51 +02:00
Ali Oezyetimoglu
df0c344af2 RED-7241: refactored some code and deleted dead code 2023-08-14 11:00:06 +02:00
Ali Oezyetimoglu
ebb407b53c RED-7241: added many log files to debug strange behaviour 2023-08-14 10:59:56 +02:00
Ali Oezyetimoglu
c3aea5ad32 RED-7241: added many log files to debug strange behaviour 2023-08-14 10:59:46 +02:00
Ali Oezyetimoglu
cae5d386da RED-7241: added many log files to debug strange behaviour 2023-08-14 10:59:33 +02:00
Ali Oezyetimoglu
480711e39c RED-7241: fixed logic to adapt resize endpoints;
removed unused and deprecated methods
2023-08-14 10:59:25 +02:00
Ali Oezyetimoglu
6c4cd3e44c RED-7241: added flag addToAllDossiers to ManualResizeRedaction 2023-08-14 10:59:16 +02:00
Ali Oezyetimoglu
d3337d4244 RED-7241: adapted resize endpoints for multi-level dictionaries and added Junit tests and did some refactoring 2023-08-14 10:59:07 +02:00
Ali Oezyetimoglu
a561f4f2da RED-7241: cleaned up unneeded imports 2023-08-14 10:58:49 +02:00
40 changed files with 847 additions and 1614 deletions

View File

@ -116,11 +116,32 @@ public class FileManagementController implements FileManagementResource {
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "inline", required = false, defaultValue = FALSE) boolean inline) {
try {
return getResponseEntityForPDFDocument(fileId, dossierId, FileType.ORIGIN, inline);
}
@Timed
@Override
@PreAuthorize("hasAuthority('" + DOWNLOAD_ORIGINAL_FILE + "')")
public ResponseEntity<?> downloadViewerDocument(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "inline", required = false, defaultValue = FALSE) boolean inline) {
// Viewer Document Returns
if (storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT))) {
return getResponseEntityForPDFDocument(fileId, dossierId, FileType.VIEWER_DOCUMENT, inline);
} else {
return getResponseEntityForPDFDocument(fileId, dossierId, FileType.ORIGIN, inline);
}
}
private ResponseEntity<?> getResponseEntityForPDFDocument(String fileId, String dossierId, FileType viewerDocument, boolean inline) {
try {
var file = fileStatusManagementService.getFileStatus(fileId);
var untouchedFileStream = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ORIGIN));
return getResponseEntity(inline, untouchedFileStream, file.getFilename(), MediaType.APPLICATION_PDF);
var viewerDocumentFileStream = storageService.getObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, viewerDocument));
return getResponseEntity(inline, viewerDocumentFileStream, file.getFilename(), MediaType.APPLICATION_PDF);
} catch (FeignException e) {
if (e.status() == HttpStatus.NOT_FOUND.value()) {
return new ResponseEntity<>(e.getMessage(), HttpStatus.NOT_FOUND);

View File

@ -1,6 +1,10 @@
package com.iqser.red.persistence.service.v1.external.api.impl.controller;
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.*;
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.ADD_COMMENT;
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.DELETE_COMMENT;
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.DELETE_MANUAL_REDACTION;
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.DO_MANUAL_REDACTION;
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_MANUAL_REDACTIONS;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.util.ArrayList;
@ -37,7 +41,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ApproveRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequest;
@ -63,395 +66,6 @@ public class ManualRedactionController implements ManualRedactionResource {
private final AccessControlService accessControlService;
/* Reviewer Operations*/
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public ManualAddResponse requestAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody AddRedactionRequest addRedactionRequest) {
return requestBulkAddRedaction(dossierId, fileId, Set.of(addRedactionRequest)).get(0);
}
@Override
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public List<ManualAddResponse> requestBulkAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<AddRedactionRequest> addRedactionRequests) {
var dossier = dossierManagementService.getDossierById(dossierId, false, false);
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest> requests = new ArrayList<>();
for (var addRedactionRequest : addRedactionRequests) {
var addRedactionRequestBuilder = com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest.builder()
.user(KeycloakSecurity.getUserId())
.dossierTemplateTypeId(toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId()))
.value(addRedactionRequest.getValue())
.reason(addRedactionRequest.getReason())
.legalBasis(addRedactionRequest.getLegalBasis())
.addToDictionary(addRedactionRequest.isAddToDictionary())
.status(AnnotationStatus.REQUESTED)
.section(addRedactionRequest.getSection())
.positions(addRedactionRequest.getPositions())
.comment(addRedactionRequest.getComment() != null ? addRedactionRequest.getComment().getText() : null)
.addToAllDossiers(addRedactionRequest.isAddToAllDossiers())
.forceAddToDictionary(addRedactionRequest.isForceAddToDictionary())
.rectangle(addRedactionRequest.isRectangle())
.dictionaryEntryType(addRedactionRequest.getDictionaryEntryType())
.sourceId(addRedactionRequest.getSourceId());
requests.add(addRedactionRequestBuilder.build());
}
List<ManualAddResponse> responseList = new ArrayList<>();
responseList = manualRedactionService.addAddRedaction(dossierId, fileId, requests);
responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Manual redaction was requested.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
.build()));
return responseList;
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public ManualAddResponse requestRemoveRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody RemoveRedactionRequest removeRedactionRequest) {
return processRequestRemoveRedactionBulk(dossierId, fileId, Set.of(removeRedactionRequest)).get(0);
}
private List<ManualAddResponse> processRequestRemoveRedactionBulk(String dossierId, String fileId, Set<RemoveRedactionRequest> removeRedactionRequests) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest> requests = removeRedactionRequests.stream()
.map(removeRedactionRequest -> com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.RemoveRedactionRequest.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
.status(AnnotationStatus.REQUESTED)
.removeFromDictionary(removeRedactionRequest.isRemoveFromDictionary())
.comment(removeRedactionRequest.getComment())
.build())
.collect(Collectors.toList());
List<ManualAddResponse> responseList = manualRedactionService.addRemoveRedaction(dossierId, fileId, requests);
responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Manual removed redaction was requested.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
.build()));
return responseList;
}
@Override
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public List<ManualAddResponse> requestBulkRemoveRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<RemoveRedactionRequest> removeRedactionRequests) {
return processRequestRemoveRedactionBulk(dossierId, fileId, removeRedactionRequests);
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public ManualAddResponse requestForceRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ForceRedactionRequest forceRedactionRequest) {
return processRequestForceRedactionBulk(dossierId, fileId, Set.of(forceRedactionRequest)).get(0);
}
private List<ManualAddResponse> processRequestForceRedactionBulk(String dossierId, String fileId, Set<ForceRedactionRequest> forceRedactionRequests) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest> requests = forceRedactionRequests.stream()
.map(forceRedactionRequest -> com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ForceRedactionRequest.builder()
.annotationId(forceRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
.status(AnnotationStatus.REQUESTED)
.comment(forceRedactionRequest.getComment())
.legalBasis(forceRedactionRequest.getLegalBasis())
.build())
.collect(Collectors.toList());
List<ManualAddResponse> responseList = manualRedactionService.addForceRedaction(dossierId, fileId, requests);
responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Manual force redaction was requested.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
.build()));
return responseList;
}
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public List<ManualAddResponse> requestBulkForceRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ForceRedactionRequest> forceRedactionRequests) {
return processRequestForceRedactionBulk(dossierId, fileId, forceRedactionRequests);
}
@Deprecated
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public ManualAddResponse requestLegalBasisChange(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody LegalBasisChangeRequest legalBasisChangeRequest) {
return processRequestLegalBasisChangeBulk(dossierId, fileId, Set.of(legalBasisChangeRequest)).get(0);
}
private List<ManualAddResponse> processRequestLegalBasisChangeBulk(String dossierId, String fileId, Set<LegalBasisChangeRequest> legalBasisChangeRequests) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest> requests = legalBasisChangeRequests.stream()
.map(legalBasisChangeRequest -> com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.LegalBasisChangeRequest.builder()
.annotationId(legalBasisChangeRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
.status(AnnotationStatus.REQUESTED)
.section(legalBasisChangeRequest.getSection())
.comment(legalBasisChangeRequest.getComment())
.legalBasis(legalBasisChangeRequest.getLegalBasis())
.value(legalBasisChangeRequest.getValue())
.build())
.collect(Collectors.toList());
List<ManualAddResponse> responseList = manualRedactionService.addLegalBasisChange(dossierId, fileId, requests);
responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Manual force redaction was requested.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
.build()));
return responseList;
}
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public List<ManualAddResponse> requestBulkLegalBasisChange(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<LegalBasisChangeRequest> legalBasisChangeRequests) {
return processRequestLegalBasisChangeBulk(dossierId, fileId, legalBasisChangeRequests);
}
@Deprecated
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public ManualAddResponse requestImageRecategorization(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ImageRecategorizationRequest imageRecategorizationRequest) {
return processRequestImageRecategorizationBulk(dossierId, fileId, Set.of(imageRecategorizationRequest)).get(0);
}
private List<ManualAddResponse> processRequestImageRecategorizationBulk(String dossierId, String fileId, Set<ImageRecategorizationRequest> imageRecategorizationRequests) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
var dossier = dossierManagementService.getDossierById(dossierId, false, false);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ImageRecategorizationRequest> requests = imageRecategorizationRequests.stream()
.map(imageRecategorizationRequest -> com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ImageRecategorizationRequest.builder()
.annotationId(imageRecategorizationRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
.status(AnnotationStatus.REQUESTED)
.typeId(toTypeId(imageRecategorizationRequest.getType(), dossier.getDossierTemplateId()))
.comment(imageRecategorizationRequest.getComment())
.build())
.collect(Collectors.toList());
List<ManualAddResponse> responseList = manualRedactionService.addImageRecategorization(dossierId, fileId, requests);
responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Image recategorization was requested.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
.build()));
return responseList;
}
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public List<ManualAddResponse> requestBulkImageRecategorization(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ImageRecategorizationRequest> imageRecategorizationRequests) {
return processRequestImageRecategorizationBulk(dossierId, fileId, imageRecategorizationRequests);
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public ManualAddResponse requestResizeRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ResizeRedactionRequest resizeRedactionRequest) {
return processRequestResizeRedactionBulk(dossierId, fileId, Set.of(resizeRedactionRequest)).get(0);
}
private List<ManualAddResponse> processRequestResizeRedactionBulk(String dossierId, String fileId, Set<ResizeRedactionRequest> resizeRedactionRequests) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest> requests = resizeRedactionRequests.stream()
.map(resizeRedactionRequest -> com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest.builder()
.annotationId(resizeRedactionRequest.getAnnotationId())
.user(KeycloakSecurity.getUserId())
.status(AnnotationStatus.REQUESTED)
.comment(resizeRedactionRequest.getComment())
.positions(resizeRedactionRequest.getPositions())
.value(resizeRedactionRequest.getValue())
.updateDictionary(resizeRedactionRequest.getUpdateDictionary())
.build())
.collect(Collectors.toList());
List<ManualAddResponse> responseList = manualRedactionService.addResizeRedaction(dossierId, fileId, requests);
responseList.forEach(response -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Manual resize redaction was requested.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, response.getAnnotationId()))
.build()));
return responseList;
}
@PreAuthorize("hasAuthority('" + REQUEST_MANUAL_REDACTION + "')")
public List<ManualAddResponse> requestBulkResizeRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ResizeRedactionRequest> resizeRedactionRequests) {
return processRequestResizeRedactionBulk(dossierId, fileId, resizeRedactionRequests);
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + DELETE_MANUAL_REDACTION + "')")
public void undo(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @PathVariable(ANNOTATION_ID) String annotationId) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsReviewer(dossierId, fileId);
ManualRedactions manualRedactions = manualRedactionService.getManualRedactions(fileId);
ManualRedactionWrapper manualRedactionWrapper = getLatestManualRedactionForAnnotationId(manualRedactions, annotationId);
if (manualRedactionWrapper == null) {
throw new NotFoundException(String.format("ManualRedaction with annotationId %s could not be found.", annotationId));
}
if (manualRedactionWrapper.getItem() instanceof ManualRedactionEntry) {
manualRedactionService.deleteAddRedaction(dossierId, fileId, List.of(annotationId));
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual add redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
} else if (manualRedactionWrapper.getItem() instanceof IdRemoval) {
manualRedactionService.deleteRemoveRedaction(dossierId, fileId, List.of(annotationId));
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual remove redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
} else if (manualRedactionWrapper.getItem() instanceof ManualForceRedaction) {
manualRedactionService.deleteForceRedaction(dossierId, fileId, List.of(annotationId));
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual force redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
} else if (manualRedactionWrapper.getItem() instanceof ManualImageRecategorization) {
manualRedactionService.deleteImageRecategorization(dossierId, fileId, List.of(annotationId));
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual image recategorization was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
} else if (manualRedactionWrapper.getItem() instanceof ManualLegalBasisChange) {
manualRedactionService.deleteLegalBasisChange(dossierId, fileId, List.of(annotationId));
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of legal basis change was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
} else if (manualRedactionWrapper.getItem() instanceof ManualResizeRedaction) {
manualRedactionService.deleteResizeRedaction(dossierId, fileId, List.of(annotationId));
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual resize redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build());
}
}
private ManualRedactionWrapper getLatestManualRedactionForAnnotationId(ManualRedactions manualRedactions, String annotationId) {
@ -649,150 +263,6 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + PROCESS_MANUAL_REDACTION_REQUEST + "')")
public void declineRequest(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @PathVariable(ANNOTATION_ID) String annotationId) {
declineRequestBulk(dossierId, fileId, Set.of(annotationId));
}
/* Approver Operations*/
@PreAuthorize("hasAuthority('" + PROCESS_MANUAL_REDACTION_REQUEST + "')")
public void declineRequestBulk(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<String> annotationIds) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsApprover(dossierId);
updateAnnotationStatus(dossierId, fileId, annotationIds, AnnotationStatus.DECLINED);
annotationIds.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Suggestion was declined")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
private void updateAnnotationStatus(String dossierId, String fileId, Set<String> annotationIds, AnnotationStatus status) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
ManualRedactions manualRedactions = manualRedactionService.getManualRedactions(fileId);
Map<String, ManualRedactionWrapper> manualRedactionWrappers = getLatestManualRedactionsForAnnotationIds(manualRedactions, annotationIds);
if (manualRedactionWrappers.isEmpty()) {
throw new NotFoundException(String.format("ManualRedaction with annotationIds %s could not be found.", annotationIds));
}
List<String> manualRedactionEntries = manualRedactionWrappers.values()
.stream()
.filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualRedactionEntry)
.map(ManualRedactionWrapper::getId)
.collect(Collectors.toList());
if (!manualRedactionEntries.isEmpty()) {
manualRedactionService.updateAddRedactionStatus(dossierId, fileId, manualRedactionEntries, status);
manualRedactionEntries.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual add redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
List<String> idRemovals = manualRedactionWrappers.values()
.stream()
.filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof IdRemoval)
.map(ManualRedactionWrapper::getId)
.collect(Collectors.toList());
if (!idRemovals.isEmpty()) {
manualRedactionService.updateRemoveRedactionStatus(dossierId, fileId, idRemovals, status);
idRemovals.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual remove redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
List<String> manualForceRedactions = manualRedactionWrappers.values()
.stream()
.filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualForceRedaction)
.map(ManualRedactionWrapper::getId)
.collect(Collectors.toList());
if (!manualForceRedactions.isEmpty()) {
manualRedactionService.updateForceRedactionStatus(dossierId, fileId, manualForceRedactions, status);
manualForceRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual force redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
List<String> manualImageRecategorizations = manualRedactionWrappers.values()
.stream()
.filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualImageRecategorization)
.map(ManualRedactionWrapper::getId)
.collect(Collectors.toList());
if (!manualImageRecategorizations.isEmpty()) {
manualRedactionService.updateImageRecategorizationStatus(dossierId, fileId, manualImageRecategorizations, status);
manualImageRecategorizations.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual image recategorization was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
List<String> manualLegalBasisChanges = manualRedactionWrappers.values()
.stream()
.filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualLegalBasisChange)
.map(ManualRedactionWrapper::getId)
.collect(Collectors.toList());
if (!manualLegalBasisChanges.isEmpty()) {
manualRedactionService.updateLegalBasisChangeStatus(dossierId, fileId, manualLegalBasisChanges, status);
manualLegalBasisChanges.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of legal basis change was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
List<String> manualResizeRedactions = manualRedactionWrappers.values()
.stream()
.filter(manualRedactionWrapper -> manualRedactionWrapper.getItem() instanceof ManualResizeRedaction)
.map(ManualRedactionWrapper::getId)
.collect(Collectors.toList());
if (!manualResizeRedactions.isEmpty()) {
manualRedactionService.updateResizeRedactionStatus(dossierId, fileId, manualResizeRedactions, status);
manualResizeRedactions.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Undo of manual resize redaction was done.")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
}
@Override
@PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
public ManualRedactions getManualRedactions(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
@ -827,45 +297,6 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + PROCESS_MANUAL_REDACTION_REQUEST + "')")
public void approveRequest(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@PathVariable(ANNOTATION_ID) String annotationId,
@RequestBody ApproveRequest approveRequest) {
approveRequestBulk(dossierId, fileId, Set.of(annotationId));
}
@PreAuthorize("hasAuthority('" + PROCESS_MANUAL_REDACTION_REQUEST + "')")
public void approveRequestBulk(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<String> annotationIds) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsApprover(dossierId);
updateAnnotationStatus(dossierId, fileId, annotationIds, AnnotationStatus.APPROVED);
annotationIds.forEach(annotationId -> auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)
.category(AuditCategory.DOCUMENT.name())
.message("Suggestion was approved")
.details(Map.of(DOSSIER_ID, dossierId, FILE_ID, fileId, ANNOTATION_ID, annotationId))
.build()));
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public ManualAddResponse addRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AddRedactionRequest addRedactionRequest) {
return addRedactionBulk(dossierId, fileId, Set.of(addRedactionRequest)).get(0);
}
@Override
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public List<ManualAddResponse> addRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@ -917,17 +348,6 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public ManualAddResponse removeRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody RemoveRedactionRequest removeRedactionRequest) {
return removeRedactionBulk(dossierId, fileId, Set.of(removeRedactionRequest)).get(0);
}
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public List<ManualAddResponse> removeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@ -970,17 +390,6 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public ManualAddResponse forceRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ForceRedactionRequest forceRedactionRequest) {
return forceRedactionBulk(dossierId, fileId, Set.of(forceRedactionRequest)).get(0);
}
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public List<ManualAddResponse> forceRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@ -1012,16 +421,6 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public ManualAddResponse legalBasisChange(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody LegalBasisChangeRequest legalBasisChangeRequest) {
return legalBasisChangeBulk(dossierId, fileId, Set.of(legalBasisChangeRequest)).get(0);
}
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public List<ManualAddResponse> legalBasisChangeBulk(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@ -1056,16 +455,6 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public ManualAddResponse recategorizeImage(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ImageRecategorizationRequest imageRecategorizationRequest) {
return recategorizeImageBulk(dossierId, fileId, Set.of(imageRecategorizationRequest)).get(0);
}
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public List<ManualAddResponse> recategorizeImageBulk(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@ -1099,22 +488,10 @@ public class ManualRedactionController implements ManualRedactionResource {
}
@Deprecated
@Override
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public ManualAddResponse resizeRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ResizeRedactionRequest resizeRedactionRequest) {
return resizeRedactionBulk(dossierId, fileId, Set.of(resizeRedactionRequest)).get(0);
}
@PreAuthorize("hasAuthority('" + DO_MANUAL_REDACTION + "')")
public List<ManualAddResponse> resizeRedactionBulk(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ResizeRedactionRequest> resizeRedactionRequests) {
accessControlService.verifyFileIsNotApproved(dossierId, fileId);
accessControlService.verifyUserIsMemberOrApprover(dossierId);
@ -1127,6 +504,7 @@ public class ManualRedactionController implements ManualRedactionResource {
.value(resizeRedactionRequest.getValue())
.comment(resizeRedactionRequest.getComment())
.updateDictionary(resizeRedactionRequest.getUpdateDictionary())
.addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
.build())
.collect(Collectors.toList());

View File

@ -470,6 +470,7 @@ public class StatusController implements StatusResource {
.rulesVersion(status.getRulesVersion())
.legalBasisVersion(status.getLegalBasisVersion())
.lastProcessed(status.getLastProcessed())
.lastLayoutProcessed(status.getLastLayoutProcessed())
.approvalDate(status.getApprovalDate())
.lastUploaded(status.getLastUploaded())
.analysisDuration(status.getAnalysisDuration())

View File

@ -25,6 +25,7 @@ public interface FileManagementResource {
String DELETE_PATH = ExternalApi.BASE_PATH + "/delete";
String DOWNLOAD_ORIGINAL_PATH = ExternalApi.BASE_PATH + "/download/original";
String DOWNLOAD_VIEWER_DOCUMENT_PATH = ExternalApi.BASE_PATH + "/download/viewer_doc";
String ROTATION_PATH = ExternalApi.BASE_PATH + "/rotate";
String DOSSIER_ID = "dossierId";
@ -55,14 +56,24 @@ public interface FileManagementResource {
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@Operation(summary = "Returns a downloadable byte stream of the original file with the specified fileId", description = "Use the optional \"inline\" request parameter to select, if " + "this downloadAnnotated will be opened in the browser.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Could not " + "prepare file download.")})
@Operation(summary = "Returns a downloadable byte stream of the original file with the specified fileId", description = "Use the optional \"inline\" request parameter to select, if this downloadAnnotated will be opened in the browser.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Could not prepare file download.")})
@GetMapping(value = DOWNLOAD_ORIGINAL_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE)
ResponseEntity<?> downloadOriginal(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "inline", required = false, defaultValue = FALSE) boolean inline);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@Operation(summary = "Returns a downloadable byte stream of the viewer document file with the specified fileId", description = "Use the optional \"inline\" request parameter to select, if this downloadAnnotated will be opened in the browser.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Could not prepare file download.")})
@GetMapping(value = DOWNLOAD_VIEWER_DOCUMENT_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE)
ResponseEntity<?> downloadViewerDocument(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "inline", required = false, defaultValue = FALSE) boolean inline);
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@DeleteMapping(value = HARD_DELETE_PATH + DOSSIER_ID_PATH_VARIABLE)
@Operation(summary = "Hard deletes an uploaded file.", description = "None")

View File

@ -17,7 +17,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ApproveRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequest;
@ -45,129 +44,6 @@ public interface ManualRedactionResource {
String COMMENT_ID_PATH_VARIABLE = "/{" + COMMENT_ID + "}";
/* Reviewer Operations*/
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/request/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request for a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse requestAddRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AddRedactionRequest addRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/request/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request for a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
List<ManualAddResponse> requestBulkAddRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<AddRedactionRequest> addRedactionRequest);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/request/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request to remove a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse requestRemoveRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody RemoveRedactionRequest removeRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/request/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a bulk of requests to remove a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
List<ManualAddResponse> requestBulkRemoveRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<RemoveRedactionRequest> removeRedactionRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/request/force" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request to force a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse requestForceRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ForceRedactionRequest forceRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/request/force" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a bulk of requests to force a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
List<ManualAddResponse> requestBulkForceRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ForceRedactionRequest> forceRedactionRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/request/legalBasis" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request to change the legal basis reason.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse requestLegalBasisChange(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody LegalBasisChangeRequest legalBasisChangeRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/request/legalBasis" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a bulk of requests to change the legal basis reason.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
List<ManualAddResponse> requestBulkLegalBasisChange(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<LegalBasisChangeRequest> legalBasisChangeRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/request/recategorize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request to recategorize a image.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse requestImageRecategorization(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ImageRecategorizationRequest imageRecategorizationRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/request/recategorize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a bulk of requests to recategorize a image.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
List<ManualAddResponse> requestBulkImageRecategorization(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ImageRecategorizationRequest> imageRecategorizationRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/request/resize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a request to resize a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse requestResizeRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ResizeRedactionRequest resizeRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/request/resize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a bulk of requests to resize a redaction.", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
List<ManualAddResponse> requestBulkResizeRedaction(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody Set<ResizeRedactionRequest> resizeRedactionRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@DeleteMapping(MANUAL_REDACTION_REST_PATH + "/undo" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + ANNOTATION_ID_PATH_VARIABLE)
@Operation(summary = "Undo a manual request or redaction", description = "Can only be done be the " + "user who added the request/redaction.")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
void undo(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @PathVariable(ANNOTATION_ID) String annotationId);
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@DeleteMapping(MANUAL_REDACTION_REST_PATH + "/bulk/undo" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)
@Operation(summary = "Undo a list of manual requests or redactions", description = "Can only be done be the " + "user who added the request/redaction.")
@ -195,50 +71,6 @@ public interface ManualRedactionResource {
@PathVariable(COMMENT_ID) String commentId);
/* Approver Operations*/
@Deprecated
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/approve" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + ANNOTATION_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Approves a redaction request/ remove redaction request", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
void approveRequest(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@PathVariable(ANNOTATION_ID) String annotationId,
@RequestBody ApproveRequest approveRequest);
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/approve" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Approves a list of redaction requests/ remove redaction requests", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
void approveRequestBulk(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<String> annotationIds);
@Deprecated
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/decline" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + ANNOTATION_ID_PATH_VARIABLE)
@Operation(summary = "Declines a redaction request/ remove redaction request", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
void declineRequest(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @PathVariable(ANNOTATION_ID) String annotationId);
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/decline" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)
@Operation(summary = "Declines a redaction request/ remove redaction request list", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK")})
void declineRequestBulk(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<String> annotationIds);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/redaction/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a manual redaction", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse addRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AddRedactionRequest addRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/add" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Adds a manual redaction", description = "None")
@ -248,14 +80,6 @@ public interface ManualRedactionResource {
@RequestBody Set<AddRedactionRequest> addRedactionRequest);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/redaction/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Removes a redaction", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse removeRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody RemoveRedactionRequest removeRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/remove" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Removes the redactions list", description = "None")
@ -265,14 +89,6 @@ public interface ManualRedactionResource {
@RequestBody Set<RemoveRedactionRequest> removeRedactionRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/redaction/force" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Forces a redaction", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse forceRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody ForceRedactionRequest forceRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/force" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Forces the redactions list", description = "None")
@ -282,16 +98,6 @@ public interface ManualRedactionResource {
@RequestBody Set<ForceRedactionRequest> forceRedactionRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/redaction/legalBasisChange" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Changes a legal basis reason", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse legalBasisChange(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody LegalBasisChangeRequest legalBasisChangeRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/legalBasisChange" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Changes the legal basis reasons list", description = "None")
@ -301,16 +107,6 @@ public interface ManualRedactionResource {
@RequestBody Set<LegalBasisChangeRequest> legalBasisChangeRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/redaction/recategorize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Recategorizes an image", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse recategorizeImage(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestBody ImageRecategorizationRequest imageRecategorizationRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/recategorize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Recategorizes the images list", description = "None")
@ -320,14 +116,6 @@ public interface ManualRedactionResource {
@RequestBody Set<ImageRecategorizationRequest> imageRecategorizationRequests);
@Deprecated
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/redaction/resize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Resizes a redaction", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ManualAddResponse resizeRedaction(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody ResizeRedactionRequest resizeRedactionRequest);
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = MANUAL_REDACTION_REST_PATH + "/bulk/redaction/resize" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Resizes the redactions list", description = "None")
@ -337,8 +125,6 @@ public interface ManualRedactionResource {
@RequestBody Set<ResizeRedactionRequest> resizeRedactionRequests);
/* Other operations */
@ResponseStatus(value = HttpStatus.OK)
@GetMapping(value = MANUAL_REDACTION_REST_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)

View File

@ -49,6 +49,7 @@ public interface RedactionLogResource {
@RequestParam(value = "includeFalsePositives", required = false, defaultValue = "false") boolean includeFalsePositives);
@Deprecated
@GetMapping(value = SECTION_GRID_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Gets the section grid for a fileId", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The section grid is not found.")})

View File

@ -91,7 +91,7 @@
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service-internal-api</artifactId>
<version>0.19.0</version>
<version>0.37.0</version>
</dependency>
<dependency>

View File

@ -2,7 +2,15 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import jakarta.persistence.Column;
import jakarta.persistence.ElementCollection;
@ -13,13 +21,6 @@ import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -65,4 +66,10 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
@Column
private String textAfter;
@Column
private boolean addToAllDossiers;
@ElementCollection
@Fetch(value = FetchMode.SUBSELECT)
private Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
}

View File

@ -70,6 +70,9 @@ public class FileEntity {
@Column
private OffsetDateTime lastProcessed;
@Column
private OffsetDateTime lastLayoutProcessed;
@Column
private OffsetDateTime lastIndexed;

View File

@ -14,8 +14,6 @@ import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jakarta.transaction.Transactional;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -29,13 +27,14 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotAl
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.TextNormalizationUtilities;
import com.iqser.red.service.persistence.management.v1.processor.utils.TypeMapper;
import com.iqser.red.service.persistence.management.v1.processor.validation.DictionaryValidator;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

View File

@ -39,12 +39,7 @@ public class FileStatusProcessingUpdateService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
switch (analyzeResult.getMessageType()) {
case SURROUNDING_TEXT:
fileStatusService.setStatusProcessed(analyzeResult.getFileId());
manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions());
break;
case ANALYSE:
case REANALYSE:
default:

View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@ -83,7 +84,7 @@ public class FileStatusService {
var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries());
var convertedList = MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper());
log.info("Initial list of files for scheduler contains: {} files.", fileEntities.size());
log.debug("Initial list of files for scheduler contains: {} files.", fileEntities.size());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
}
@ -717,6 +718,12 @@ public class FileStatusService {
}
public void updateLayoutProcessedTime(String fileId) {
fileStatusPersistenceService.updateLayoutProcessedTime(fileId);
}
public int countSoftDeletedFiles(String dossierId) {
return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId);

View File

@ -40,7 +40,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AddRedactionRequest;
@ -59,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
@ -70,7 +70,6 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionService {
private final DossierPersistenceService dossierPersistenceService;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final RemoveRedactionPersistenceService removeRedactionPersistenceService;
private final ForceRedactionPersistenceService forceRedactionPersistenceService;
@ -85,8 +84,6 @@ public class ManualRedactionService {
private final ManualRedactionProviderService manualRedactionProviderService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final StopwordService stopwordService;
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final RedactionLogService redactionLogService;
private final DictionaryManagementService dictionaryManagementService;
private final HashFunction hashFunction = Hashing.murmur3_128();
@ -98,10 +95,13 @@ public class ManualRedactionService {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
// validate add to dossier template dictionaries
addRedactionRequests.forEach(request -> dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(request.getDossierTemplateTypeId(), request.isAddToDictionary(), request.isAddToAllDossiers()));
// validate add to dossier template dictionaries
addRedactionRequests.forEach(request -> dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(request.getDossierTemplateTypeId(),
request.isAddToDictionary(),
request.isAddToAllDossiers()));
boolean actionPerformed = false;
for (var addRedactionRequest : addRedactionRequests) {
if (addRedactionRequest.isAddToDictionary()) {
validateDictionary(addRedactionRequest);
@ -116,7 +116,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
var addedToDictionary = handleAddToDictionary(fileId,
handleAddToDictionary(fileId,
annotationId,
addRedactionRequest.getDossierTemplateTypeId(),
addRedactionRequest.getValue(),
@ -127,28 +127,14 @@ public class ManualRedactionService {
dossierId,
addRedactionRequest.getDictionaryEntryType());
actionPerformed = actionPerformed || addedToDictionary;
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
actionPerformed = actionPerformed || addRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId);
} else {
var manualTextRedactions = response.stream()
.map(r -> manualRedactionProviderService.getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isRectangle())
.toList();
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
}
return response;
@ -242,20 +228,6 @@ public class ManualRedactionService {
}
private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);
var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build();
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, analyseRequest, message -> {
message.getMessageProperties().setPriority(1);
return message;
});
}
private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) {
try {
@ -280,21 +252,21 @@ public class ManualRedactionService {
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
var actionPerformed = false;
var requiresReAnalysis = false;
var manualRedactions = manualRedactionProviderService.getManualRedactions(fileId);
//validate removing from dossier template dictionary
removeRedactionRequests.forEach(request -> {
removeRedactionRequests.forEach(request -> {
if (request.isRemoveFromDictionary()) {
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, request.getAnnotationId());
var dossierTemplateTypeId = toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(dossierTemplateTypeId, request.isRemoveFromDictionary(), request.isRemoveFromAllDossiers());
dictionaryManagementService.validateAddRemoveToDossierTemplateDictionary(dossierTemplateTypeId,
request.isRemoveFromDictionary(),
request.isRemoveFromAllDossiers());
}
});
@ -312,18 +284,16 @@ public class ManualRedactionService {
if (removeRedactionRequest.getComment() != null) {
commentId = addComment(fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getComment(), removeRedactionRequest.getUser()).getId();
}
if (!removeRedactionRequest.isRemoveFromDictionary() && AnnotationStatus.APPROVED.equals(removeRedactionRequest.getStatus())) {
RedactionLogEntry redactionLogEntry = null;
boolean requiresAnalysis = false;
boolean matchingEntryFound;
try {
redactionLogEntry = getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
requiresAnalysis = redactionLogEntry.isHint();
getRedactionLogEntry(redactionLog, removeRedactionRequest.getAnnotationId());
matchingEntryFound = true;
} catch (NotFoundException e) {
requiresAnalysis = false;
matchingEntryFound = false;
}
actionPerformed = actionPerformed || requiresAnalysis;
if (!requiresAnalysis && idRemoval.isApproved()) {
requiresReAnalysis = requiresReAnalysis || matchingEntryFound;
if (!matchingEntryFound && idRemoval.isApproved()) {
removeRedactionPersistenceService.markAsProcessed(idRemoval);
}
}
@ -341,13 +311,13 @@ public class ManualRedactionService {
removeRedactionPersistenceService.markAsProcessed(idRemoval);
}
actionPerformed = actionPerformed || removedFromDictionary;
requiresReAnalysis = requiresReAnalysis || removedFromDictionary;
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
}
if (actionPerformed) {
if (requiresReAnalysis) {
reprocess(dossierId, fileId);
}
@ -387,8 +357,8 @@ public class ManualRedactionService {
});
} else {
if (removeFromAllDossiers) {
var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(),dossier.getDossierTemplateId()),
redactionLogEntry.getValue());
var dictionaryEntriesToRemove = dictionaryManagementService.getAllEntriesInDossierTemplate(toTypeId(redactionLogEntry.getType(),
dossier.getDossierTemplateId()), redactionLogEntry.getValue());
dictionaryEntriesToRemove.forEach(entry -> {
typeIdsOfModifiedDictionaries.add(entry.getTypeId());
removeFromDictionary(entry.getTypeId(), entry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY);
@ -417,10 +387,7 @@ public class ManualRedactionService {
private RedactionLogEntry getRedactionLogEntry(RedactionLog redactionLog, String annotationId) {
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst();
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst();
if (redactionLogEntryOptional.isEmpty()) {
throw new NotFoundException("Annotation does not exist in redaction log.");
@ -431,16 +398,6 @@ public class ManualRedactionService {
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
if (redactionLogEntry.isDossierDictionaryEntry()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
} else {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
}
}
private void removeResizeRedactionsWithAddToDictionary(String dossierTemplateId, String redactionLogEntryValue) {
var resizeRedactionsWithSameValue = resizeRedactionPersistenceService.findByAnnotationStatusAndValue(AnnotationStatus.APPROVED, redactionLogEntryValue);
@ -469,7 +426,7 @@ public class ManualRedactionService {
commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId();
}
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus());
actionPerformed = actionPerformed || forceRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
@ -525,7 +482,7 @@ public class ManualRedactionService {
imageRecategorizationRequest.getUser()).getId();
}
actionPerformed = actionPerformed || !AnnotationStatus.REQUESTED.equals(imageRecategorizationRequest.getStatus());
actionPerformed = actionPerformed || imageRecategorizationRequest.getStatus().equals(AnnotationStatus.APPROVED);
response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build());
}
@ -718,33 +675,63 @@ public class ManualRedactionService {
private void updateDictionaryForResizeRedactions(String dossierId, String fileId, ManualResizeRedactionEntity resizeRedaction, RedactionLog redactionLog) {
RedactionLogEntry redactionLogEntry = null;
try {
redactionLogEntry = getRedactionLogEntry(redactionLog, resizeRedaction.getId().getAnnotationId());
} catch (NotFoundException e) {
return;
}
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, resizeRedaction.getId().getAnnotationId());
if (resizeRedaction.getUpdateDictionary() != null && resizeRedaction.getUpdateDictionary() && resizeRedaction.getStatus()
.equals(AnnotationStatus.APPROVED) && (redactionLogEntry.isDictionaryEntry() || redactionLogEntry.isDossierDictionaryEntry())) {
var dossier = dossierPersistenceService.findByDossierId(dossierId);
var typeId = buildTypeId(redactionLogEntry, dossier);
var typeId = buildTypeId(redactionLogEntry, resizeRedaction, dossier);
var newValue = resizeRedaction.getValue();
var oldValue = redactionLogEntry.getValue();
var dictionaryEntryType = getDictionaryEntryType(redactionLogEntry);
if (oldValue != null && oldValue.length() > newValue.length()) {
boolean isShrinking = oldValue != null && oldValue.length() > newValue.length();
Set<String> typeIdsOfModifiedDictionaries = new HashSet<>();
if (isShrinking) {
log.info("Remove old value '{}' from dictionary", oldValue);
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), oldValue, dossierId, fileId, dictionaryEntryType);
removeFromDictionary(typeId, oldValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
if (resizeRedaction.isAddToAllDossiers() && redactionLogEntry.isDictionaryEntry()) {
String dossierTemplateId = dossier.getDossierTemplateId();
var dossiersOfThisDossierTemplate = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
var type = redactionLogEntry.getType();
dossiersOfThisDossierTemplate.forEach(dossierEntity -> {
var typeIdOfDossierEntity = toTypeId(type, dossierTemplateId, dossierEntity.getId());
removeFromDictionary(typeIdOfDossierEntity, oldValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeIdOfDossierEntity);
});
}
}
log.info("Add new value '{}' to dictionary", newValue);
addToDictionary(typeId, newValue, dossierId, fileId, dictionaryEntryType);
typeIdsOfModifiedDictionaries.add(typeId);
resizeRedactionPersistenceService.updateStatus(resizeRedaction.getId().getFileId(),
resizeRedaction.getId().getAnnotationId(),
resizeRedaction.getStatus(),
typeIdsOfModifiedDictionaries);
}
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, ManualResizeRedactionEntity resizeRedaction, DossierEntity dossier) {
if (resizeRedaction.isAddToAllDossiers()) {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId());
} else {
return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId());
}
}
private DictionaryEntryType getDictionaryEntryType(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.isRecommendation() && redactionLogEntry.isFalsePositive()) {
@ -757,200 +744,12 @@ public class ManualRedactionService {
}
@Transactional
@SuppressWarnings("PMD")
public void updateRemoveRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true, true);
for (var annotationId : annotationIds) {
var idRemoval = MagicConverter.convert(removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId), IdRemoval.class);
RedactionLogEntry redactionLogEntry = getRedactionLogEntry(redactionLog, idRemoval.getAnnotationId());
if (idRemoval.isRemoveFromDictionary()) {
if (annotationStatus == AnnotationStatus.APPROVED) {
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
approveStatusForRedactionsWithSameValue(dossier, false, true, redactionLogEntry.getValue());
reprocess(dossierId, fileId);
} else if (annotationStatus == AnnotationStatus.DECLINED) {
// if it was previously approved, revert the delete
if (idRemoval.getStatus() == AnnotationStatus.APPROVED) {
addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
}
}
} else if (redactionLogEntry.isHint()) {
reprocess(dossierId, fileId);
}
removeRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary, boolean addToDossierDictionary, String value) {
List<DossierEntity> dossiers = new ArrayList<>();
if (addToDictionary) {
dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId()).getDossiers();
}
if (addToDossierDictionary) {
dossiers.add(dossier);
}
Set<String> fileIds = new HashSet<>();
for (DossierEntity d : dossiers) {
var files = fileStatusService.getDossierStatus(d.getId());
files.forEach(f -> fileIds.add(f.getId()));
}
if (!fileIds.isEmpty()) {
log.debug("Approve status for requested redactions with same value '{}' for files {}", value, fileIds);
addRedactionPersistenceService.approveStatusForRequestedRedactionsWithSameValue(fileIds, value);
}
}
@Transactional
public void updateForceRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
for (var annotationId : annotationIds) {
var forceRedaction = forceRedactionPersistenceService.findForceRedaction(fileId, annotationId);
forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
boolean isDeclined = forceRedaction.getStatus() == AnnotationStatus.REQUESTED && annotationStatus == AnnotationStatus.DECLINED;
actionPerformed = actionPerformed || !isDeclined;
if (isDeclined) {
forceRedactionPersistenceService.markAsProcessed(annotationId, fileId);
}
}
if (actionPerformed) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
public ManualRedactions getManualRedactions(String fileId) {
return manualRedactionProviderService.getManualRedactions(fileId);
}
@Transactional
public void updateLegalBasisChangeStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var annotationId : annotationIds) {
legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public void updateImageRecategorizationStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
for (var annotationId : annotationIds) {
var imageRecategorization = recategorizationPersistenceService.findRecategorization(fileId, annotationId);
recategorizationPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
boolean isDeclined = imageRecategorization.getStatus() == AnnotationStatus.REQUESTED && annotationStatus == AnnotationStatus.DECLINED;
actionPerformed = actionPerformed || !isDeclined;
if (isDeclined) {
recategorizationPersistenceService.markAsProcessed(annotationId, fileId);
}
}
if (actionPerformed) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) {
// These are marked as processed once surrounding text is computed ( TBD if this is correct ? )
manualRedactions.getEntriesToAdd()
.forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
}
@Transactional
public void updateResizeRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true, true);
for (var annotationId : annotationIds) {
var resizeRedaction = resizeRedactionPersistenceService.findResizeRedaction(fileId, annotationId);
actionPerformed = actionPerformed || !(resizeRedaction.getStatus() == AnnotationStatus.REQUESTED && annotationStatus == AnnotationStatus.DECLINED);
resizeRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
resizeRedaction.setStatus(annotationStatus);
updateDictionaryForResizeRedactions(dossierId, fileId, resizeRedaction, redactionLog);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (actionPerformed) {
reprocess(dossierId, fileId);
}
}
@SuppressWarnings("PMD")
@Transactional
public void updateAddRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
for (var annotationId : annotationIds) {
ManualRedactionEntryEntity manualRedactionEntry = addRedactionPersistenceService.findAddRedaction(fileId, annotationId);
if (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) {
if (annotationStatus == AnnotationStatus.APPROVED) {
addToDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId, manualRedactionEntry.getDictionaryEntryType());
reprocess(dossierId, fileId);
approveStatusForRedactionsWithSameValue(dossier,
manualRedactionEntry.isAddToDictionary(),
manualRedactionEntry.isAddToDossierDictionary(),
manualRedactionEntry.getValue());
} else if (annotationStatus == AnnotationStatus.DECLINED) {
// if it was previously approved, revert the add
if (manualRedactionEntry.getStatus() == AnnotationStatus.APPROVED) {
removeFromDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId, manualRedactionEntry.getDictionaryEntryType());
}
}
}
addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@Transactional
public void updateProcessedDate(String fileId, ManualRedactions manualRedactions) {

View File

@ -53,7 +53,7 @@ public class AutomaticAnalysisJob implements Job {
var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
if (redactionQueueInfo != null) {
log.info("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
tenant.getTenantId(),
MessagingConfiguration.REDACTION_QUEUE,
redactionQueueInfo.getMessageCount(),
@ -67,7 +67,7 @@ public class AutomaticAnalysisJob implements Job {
allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated));
var allStatusesIterator = allStatuses.iterator();
log.info("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
log.debug("[Tenant:{}] Files that require reanalysis: {}", TenantContext.getTenantId(), allStatuses.size());
var queuedFiles = 0;

View File

@ -52,6 +52,7 @@ public class LayoutParsingRequestFactory {
.positionBlockFileStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION))
.simplifiedTextStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT))
.sectionGridStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID))
.viewerDocumentStorageId(StorageIdUtils.getStorageId(dossierId, fileId, FileType.VIEWER_DOCUMENT))
.build();
}

View File

@ -4,8 +4,6 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
@ -19,6 +17,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.knecon.fforesight.databasetenantcommons.providers.utils.JDBCWriteUtils;
import jakarta.transaction.Transactional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

View File

@ -548,4 +548,10 @@ public class FileStatusPersistenceService {
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
public void updateLayoutProcessedTime(String fileId) {
fileRepository.updateLayoutProcessedTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
}

View File

@ -62,12 +62,6 @@ public class AddRedactionPersistenceService {
}
@Transactional
public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter) {
manualRedactionRepository.updateSurroundingText(id, textBefore, textAfter);
}
public ManualRedactionEntryEntity findAddRedaction(String fileId, String annotationId) {

View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@ -12,10 +13,11 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ResizeRedactionRequest;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -25,6 +27,8 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class ResizeRedactionPersistenceService {
private final ManualRedactionRepository manualRedactionRepository;
private final ResizeRedactionRepository resizeRedactionRepository;
@ -52,11 +56,14 @@ public class ResizeRedactionPersistenceService {
resizeRedactionRepository.updateSurroundingText(id, textBefore, textAfter);
}
@Transactional
public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) {
public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, Set<String> typeIdsOfModifiedDictionaries) {
resizeRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, OffsetDateTime.now());
var resizeRedaction = resizeRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId))
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
resizeRedaction.setStatus(annotationStatus);
resizeRedaction.setTypeIdsOfModifiedDictionaries(typeIdsOfModifiedDictionaries);
}

View File

@ -220,6 +220,12 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query(value = "update FileEntity f set f.numberOfOCRedPages = :numberOfOCRedPages, f.numberOfPagesToOCR = :numberOfPagesToOCR, f.ocrEndTime = :ocrEndTime, " + " f.lastUpdated = :lastUpdated where f.id = :fileId")
void updateOCRStatus(String fileId, int numberOfPagesToOCR, int numberOfOCRedPages, OffsetDateTime ocrEndTime, OffsetDateTime lastUpdated);
@Transactional
@Modifying(clearAutomatically = true)
@Query(value = "update FileEntity f set f.lastLayoutProcessed = :offsetDateTime where f.id = :fileId")
void updateLayoutProcessedTime(String fileId, OffsetDateTime offsetDateTime);
}

View File

@ -38,11 +38,6 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
List<ManualRedactionEntryEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.textBefore = :textBefore, m.textAfter = :textAfter where m.id = :id")
void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.status = :newStatus, m.processedDate = :processedDate where m.id.fileId in :fileIds and m.value = :filterValue and m.addToDictionary = true and m.status = :filterStatus ")
void updateStatus(Set<String> fileIds, String filterValue, AnnotationStatus filterStatus, AnnotationStatus newStatus, OffsetDateTime processedDate);

View File

@ -41,6 +41,8 @@ public class LayoutParsingFinishedMessageReceiver {
layoutParsingRequestIdentifierService.parseFileId(response.identifier()),
layoutParsingRequestIdentifierService.parsePriority(response.identifier()));
fileStatusService.updateLayoutProcessedTime(layoutParsingRequestIdentifierService.parseFileId(response.identifier()));
log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE);
}
@ -57,7 +59,7 @@ public class LayoutParsingFinishedMessageReceiver {
log.info("Failed to process layout parsing request, errorCause: {}, timestamp: {}", errorCause, timestamp);
fileStatusProcessingUpdateService.analysisFailed(layoutParsingRequestIdentifierService.parseDossierId(analyzeRequest.identifier()),
layoutParsingRequestIdentifierService.parseFileId(analyzeRequest.identifier()),
new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "redaction-service", timestamp));
new FileErrorInfo(errorCause, LayoutParsingQueueNames.LAYOUT_PARSING_DLQ, "layoutparser-service", timestamp));
}

View File

@ -102,7 +102,7 @@ public class RedactionLogMergeService {
if (excludedPages != null && !excludedPages.isEmpty()) {
entry.getPositions().forEach(pos -> {
if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
if (!isLocalManualRedaction(entry) && excludedPages.contains(pos.getPage())) {
entry.setExcluded(true);
}
});
@ -132,6 +132,16 @@ public class RedactionLogMergeService {
}
private boolean isLocalManualRedaction(RedactionLogEntry entry) {
return entry.getManualChanges() != null && entry.getManualChanges()
.stream()
.anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY || mc.getManualRedactionType() == ManualRedactionType.RESIZE && mc.getAnnotationStatus() == AnnotationStatus.APPROVED && entry.getEngines()
.contains(Engine.RULE) && !entry.getEngines().contains(Engine.DICTIONARY));
}
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();

View File

@ -151,3 +151,8 @@ databaseChangeLog:
file: db/changelog/tenant/104-add-ocr-by-default-to-dossier-template.yaml
- include:
file: db/changelog/tenant/105-add-remove-watermark-to-dossier-template.yaml
- include:
file: db/changelog/tenant/106-add-add-to-all-dossiers-to-resize-redactions.yaml
- include:
file: db/changelog/tenant/107-add-last-layout-processed-column.yaml

View File

@ -0,0 +1,46 @@
databaseChangeLog:
- changeSet:
id: add-add-to-all-dossiers-to-resize-redactions
author: ali
changes:
- addColumn:
columns:
- column:
name: add_to_all_dossiers
type: BOOLEAN
tableName: manual_resize_redaction
- changeSet:
id: add-add-to-all-dossiers-to-resize-redactions-2
author: ali
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
name: manual_resize_redaction_entity_annotation_id
type: VARCHAR(255)
- column:
constraints:
nullable: false
name: manual_resize_redaction_entity_file_id
type: VARCHAR(255)
- column:
name: type_ids_of_modified_dictionaries
type: VARCHAR(255)
tableName: manual_resize_redaction_entity_type_ids_of_modified_dictionaries
- changeSet:
id: add-add-to-all-dossiers-to-resize-redactions-3
author: ali
changes:
- addForeignKeyConstraint:
baseColumnNames: manual_resize_redaction_entity_annotation_id, manual_resize_redaction_entity_file_id
baseTableName: manual_resize_redaction_entity_type_ids_of_modified_dictionaries
constraintName: fk_resize_entity_annotation_id_file_id_for_type_ids_of_modified_dictionaries
deferrable: false
initiallyDeferred: false
onDelete: NO ACTION
onUpdate: NO ACTION
referencedColumnNames: annotation_id, file_id
referencedTableName: manual_resize_redaction
validate: true

View File

@ -0,0 +1,11 @@
databaseChangeLog:
- changeSet:
id: add-last-layout-processed-column-to-file
author: timo
changes:
- addColumn:
columns:
- column:
name: last_layout_processed
type: TIMESTAMP WITHOUT TIME ZONE
tableName: file

View File

@ -33,6 +33,9 @@ public class TypeProvider {
}
public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate, Dossier dossier, String typeName, boolean dossierDictionaryOnly) {
return testAndProvideType(dossierTemplate, dossier, typeName, dossierDictionaryOnly, 100);
}
public TypeValue testAndProvideType(DossierTemplateModel dossierTemplate, Dossier dossier, String typeName, boolean dossierDictionaryOnly, int rank) {
var type = new CreateTypeValue();
type.setType(typeName);
@ -42,7 +45,7 @@ public class TypeProvider {
type.setRecommendationHexColor("#aaaaaa");
type.setSkippedHexColor("#aaaaaa");
type.setHint(false);
type.setRank(100);
type.setRank(rank);
type.setRecommendation(false);
type.setCaseInsensitive(true);
type.setDossierTemplateId(dossierTemplate.getId());
@ -53,7 +56,7 @@ public class TypeProvider {
var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getDossierTemplateId(),dossier != null ? dossier.getId() : null,false);
var foundType =allTypes.getTypes().stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findAny();
assertThat(foundType.isPresent()).isTrue();
assertThat(foundType).isPresent();
return foundType.get();
}

View File

@ -420,11 +420,11 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
fileClient.setStatusUnderReview(dossier.getId(), file.getId(), userId);
var addRedaction = manualRedactionClient.requestBulkAddRedaction(dossierId,
var addRedaction = manualRedactionClient.addRedactionBulk(dossierId,
fileId,
Set.of(AddRedactionRequest.builder()
.addToDictionary(true)
.addToDossierDictionary(false)
.addToAllDossiers(true)
.comment(new AddCommentRequest("comment"))
.type(type.getType())
.reason("1")

View File

@ -103,6 +103,8 @@ public class FileStatus {
private boolean excluded;
@Schema(description = "Shows the last date of a successful analysis.")
private OffsetDateTime lastProcessed;
@Schema(description = "Shows the last date of a layout parsing.")
private OffsetDateTime lastLayoutProcessed;
@Schema(description = "Shows the date of approval, if approved.")
private OffsetDateTime approvalDate;
@Schema(description = "Shows last date the document was uploaded.")

View File

@ -3,8 +3,5 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model;
public enum MessageType {
ANALYSE,
REANALYSE,
STRUCTURE_ANALYSE,
SURROUNDING_TEXT
REANALYSE
}

View File

@ -10,6 +10,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
public class ManualAddResponse {
private String annotationId;
private Long commentId;

View File

@ -28,4 +28,6 @@ public class ResizeRedactionRequest {
private String textBefore;
private String textAfter;
private boolean addToAllDossiers;
}

View File

@ -22,6 +22,7 @@ public class ManualResizeRedaction extends BaseAnnotation {
private String textBefore;
private String textAfter;
private Boolean updateDictionary;
private boolean addToAllDossiers;
@Builder

View File

@ -31,6 +31,7 @@ public class FileModel {
private OffsetDateTime deleted;
private OffsetDateTime lastProcessed;
private OffsetDateTime lastIndexed;
private OffsetDateTime lastLayoutProcessed;
private OffsetDateTime lastManualChangeDate;
private int numberOfAnalyses;
private String assignee;

View File

@ -6,6 +6,7 @@ public enum FileType {
UNTOUCHED(".pdf"),
ORIGIN(".pdf"),
VIEWER_DOCUMENT(".pdf"),
REDACTION_LOG(".json"),
SIMPLIFIED_TEXT(".json"),
SECTION_GRID(".json"),

View File

@ -12,7 +12,6 @@ public enum ProcessingStatus {
PROCESSED,
PROCESSING,
REPROCESS,
SURROUNDING_TEXT_PROCESSING,
UNPROCESSED,
FULL_PROCESSING,
PRE_PROCESSING_QUEUED,
@ -20,5 +19,4 @@ public enum ProcessingStatus {
PRE_PROCESSED,
FIGURE_DETECTION_ANALYZING,
TABLE_PARSING_ANALYZING
}

View File

@ -38,9 +38,6 @@ public class AddRedactionRequest {
private AddCommentRequest comment;
@Deprecated // Do not use this anymore. will be removed.
private boolean addToDossierDictionary;
private boolean forceAddToDictionary;
private String section;

View File

@ -26,6 +26,8 @@ public class ResizeRedactionRequest {
private Boolean updateDictionary;
private boolean addToAllDossiers;
@Builder.Default
private List<Rectangle> positions = new ArrayList<>();

View File

@ -84,14 +84,6 @@ public class RedactionLogEntry {
}
public boolean isLocalManualRedaction() {
return manualChanges != null && manualChanges.stream()
.anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED
|| mc.getManualRedactionType() == ManualRedactionType.RESIZE && mc.getAnnotationStatus() == AnnotationStatus.APPROVED && engines.contains(Engine.RULE) && !engines.contains(Engine.DICTIONARY));
}
public boolean isManuallyRemoved() {
return manualChanges != null && manualChanges.stream()