Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

DM-504: always exclude hard deleted files #170

Merged
kilian.schuettler1 merged 3 commits from DM-504 into master 2023-10-10 11:32:15 +02:00
Conversation 7 Commits 3 Files Changed 10 +111 -92
10 changed files with 111 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java vendored
View File

@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionUndoService;
@ -66,6 +67,7 @@ public class ManualRedactionController implements ManualRedactionResource {
AccessControlService accessControlService;
ManualRedactionMapper manualRedactionMapper;
CommentService commentService;
FileStatusManagementService fileStatusManagementService;
@Override
@ -113,6 +115,9 @@ public class ManualRedactionController implements ManualRedactionResource {
@PreAuthorize("hasAuthority('" + READ_MANUAL_REDACTIONS + "')")
public Comments getComments(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
dossierManagementService.getDossierById(dossierId, false, false);
fileStatusManagementService.getFileStatus(fileId, false);
return commentService.getComments(fileId);
}

76
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/StatusController.java vendored
View File

@ -28,12 +28,12 @@ import com.iqser.red.service.persistence.management.v1.processor.roles.Applicati
import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.external.resource.StatusResource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
@ -90,7 +90,7 @@ public class StatusController implements StatusResource {
List<FileStatus> statusList = fileStatusManagementService.getDossierStatus(dossierId)
.stream()
.filter(fileStatus -> !fileStatus.isSoftOrHardDeleted())
.map(this::convert)
.map(FileStatusMapper::toFileStatus)
.collect(Collectors.toList());
response.put(dossierId, statusList);
} catch (AccessDeniedException e) {
@ -120,9 +120,11 @@ public class StatusController implements StatusResource {
if (dossiersWithViewPermissions.isEmpty()) {
return new HashMap<>();
}
statusList = fileStatusManagementService.getSoftDeletedForDossierList(dossiersWithViewPermissions).stream().map(this::convert).collect(Collectors.toList());
return fileStatusManagementService.getSoftDeletedForDossierList(dossiersWithViewPermissions)
.stream()
.map(FileStatusMapper::toFileStatus)
.collect(Collectors.groupingBy(FileStatus::getDossierId, Collectors.toList()));
return statusList.stream().collect(Collectors.groupingBy(FileStatus::getDossierId, Collectors.toList()));
}
@ -135,7 +137,7 @@ public class StatusController implements StatusResource {
return fileStatusManagementService.getDossierStatus(dossierId)
.stream()
.filter(fileStatus -> !fileStatus.isSoftOrHardDeleted())
.map(this::convert)
.map(FileStatusMapper::toFileStatus)
.collect(Collectors.toList());
} catch (AccessDeniedException e) {
@ -148,7 +150,7 @@ public class StatusController implements StatusResource {
@PreAuthorize("hasAuthority('" + READ_FILE_STATUS + "')")
public FileStatus getFileStatus(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId) {
return convert(fileStatusManagementService.getFileStatus(fileId));
return FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(fileId));
}
@ -421,7 +423,7 @@ public class StatusController implements StatusResource {
try {
accessControlService.verifyUserHasViewPermissions(dossierId);
return fileStatusManagementService.getSoftDeletedDossierStatus(dossierId).stream().map(this::convert).collect(Collectors.toList());
return fileStatusManagementService.getSoftDeletedDossierStatus(dossierId).stream().map(FileStatusMapper::toFileStatus).collect(Collectors.toList());
} catch (AccessDeniedException e) {
return new ArrayList<>();
@ -441,64 +443,4 @@ public class StatusController implements StatusResource {
fileStatusManagementService.setStatusApproved(dossierId, fileId, KeycloakSecurity.getUserId());
}
private FileStatus convert(FileModel status) {
return FileStatus.builder()
.dossierId(status.getDossierId())
.dossierArchived(status.isDossierArchived())
.dossierStatusId(status.getDossierStatusId())
.dossierTemplateId(status.getDossierTemplateId())
.fileId(status.getId())
.filename(status.getFilename())
.processingStatus(ProcessingStatus.valueOf(status.getProcessingStatus().name()))
.workflowStatus(WorkflowStatus.valueOf(status.getWorkflowStatus().name()))
.numberOfPages(status.getNumberOfPages())
.added(status.getAdded())
.lastUpdated(status.getLastUpdated())
.numberOfAnalyses(status.getNumberOfAnalyses())
.assignee(status.getAssignee())
.lastReviewer(status.getLastReviewer())
.lastApprover(status.getLastApprover())
.hasUpdates(status.isHasUpdates())
.hasImages(status.isHasImages())
.hasRequests(status.isHasSuggestions())
.hasSuggestions(status.isHasSuggestions())
.excludedFromAutomaticAnalysis(status.isExcludedFromAutomaticAnalysis())
.hasHints(status.isHasHints())
.hasRedactions(status.isHasRedactions())
.ocrEndTime(status.getOcrEndTime())
.ocrStartTime(status.getOcrStartTime())
.numberOfOCRedPages(status.getNumberOfOCRedPages() != null ? status.getNumberOfOCRedPages() : 0)
.numberOfPagesToOCR(status.getNumberOfPagesToOCR() != null ? status.getNumberOfPagesToOCR() : 0)
.hasAnnotationComments(status.isHasAnnotationComments())
.uploader(status.getUploader())
.dictionaryVersion(status.getDictionaryVersion())
.rulesVersion(status.getRulesVersion())
.componentRulesVersion(status.getComponentRulesVersion())
.legalBasisVersion(status.getLegalBasisVersion())
.lastProcessed(status.getLastProcessed())
.lastLayoutProcessed(status.getLastLayoutProcessed())
.approvalDate(status.getApprovalDate())
.lastUploaded(status.getLastUploaded())
.analysisDuration(status.getAnalysisDuration())
.fileAttributes(new FileAttributes(status.getFileAttributes()))
.dossierDictionaryVersion(status.getDossierDictionaryVersion())
.excluded(status.isExcluded())
.excludedPages(status.getExcludedPages())
.softDeletedTime(status.getDeleted())
.hardDeletedTime(status.getHardDeletedTime())
.analysisRequired(status.isAnalysisRequired())
.lastFileAttributeChange(status.getLastFileAttributeChange())
.redactionModificationDate(status.getRedactionModificationDate())
.fileManipulationDate(status.getFileManipulationDate())
.lastManualChangeDate(status.getLastManualChangeDate())
.hasHighlights(status.isHasHighlights())
.lastIndexed(status.getLastIndexed())
.fileSize(status.getFileSize())
.fileErrorInfo(status.getFileErrorInfo())
.build();
}
}

17
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java vendored
View File

@ -6,7 +6,6 @@ import static com.iqser.red.service.persistence.service.v2.api.external.resource
import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierResource.INCLUDE_SOFT_DELETED_PARAM;
import static com.iqser.red.service.persistence.service.v2.api.external.resource.DossierTemplateResource.DOSSIER_TEMPLATE_ID_PARAM;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -25,7 +24,8 @@ import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileAtt
import com.iqser.red.persistence.service.v1.external.api.impl.controller.FileManagementController;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.UploadController;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
@ -45,6 +45,7 @@ public class FileControllerV2 implements FileResource {
private final StatusController statusController;
private final DossierController dossierController;
private final FileManagementController fileManagementController;
private final FileStatusManagementService fileStatusManagementService;
private final FileAttributesController fileAttributesController;
private final DossierTemplateController dossierTemplateController;
@ -95,17 +96,7 @@ public class FileControllerV2 implements FileResource {
dossierController.getDossier(dossierId, true, includeSoftDeleted);
FileStatus status = statusController.getFileStatus(dossierId, fileId);
if (status.getHardDeletedTime() != null) {
throw new NotFoundException(String.format("The requested file has been hard deleted on %s.", status.getHardDeletedTime().format(DateTimeFormatter.ISO_LOCAL_DATE)));
}
if (!includeSoftDeleted && status.getSoftDeletedTime() != null) {
throw new NotFoundException(String.format("The requested file has been soft deleted on %s.", status.getSoftDeletedTime().format(DateTimeFormatter.ISO_LOCAL_DATE)));
}
return status;
return FileStatusMapper.toFileStatus(fileStatusManagementService.getFileStatus(fileId, includeSoftDeleted));
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java
View File

@ -41,7 +41,7 @@ public class EntityLogService {
}
if (excludedTypes != null) {
redactionLog.getEntityLogEntry().removeIf(nextEntry -> excludedTypes.contains(nextEntry.getType()));
redactionLog.getEntityLogEntry().removeIf(entry -> excludedTypes.contains(entry.getType()));
}
return redactionLog;
@ -68,9 +68,8 @@ public class EntityLogService {
}
}
for (var manualChange : redactionLogEntry.getManualChanges()) {
if (manualChange.getProcessedDate() != null && manualChange.getProcessedDate()
.isAfter(filteredEntityLogRequest.getSpecifiedDate()) || manualChange.getRequestedDate() != null && manualChange.getRequestedDate()
.isAfter(filteredEntityLogRequest.getSpecifiedDate())) {
if (manualChange.getProcessedDate() != null && manualChange.getProcessedDate().isAfter(filteredEntityLogRequest.getSpecifiedDate()) || //
manualChange.getRequestedDate() != null && manualChange.getRequestedDate().isAfter(filteredEntityLogRequest.getSpecifiedDate())) {
isAfterSpecifiedDate = true;
break;
}

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -8,6 +9,7 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -65,6 +67,21 @@ public class FileStatusManagementService {
}
public FileModel getFileStatus(String fileId, boolean includeSoftDeleted) {
var status = fileStatusService.getStatus(fileId);
if (status.getHardDeletedTime() != null) {
throw new NotFoundException("File not found");
}
if (!includeSoftDeleted && status.getDeleted() != null) {
throw new NotFoundException(String.format("The requested file has been soft deleted on %s.", status.getDeleted().format(DateTimeFormatter.ISO_LOCAL_DATE)));
}
return status;
}
public void updateFileModificationDate(String fileId) {
this.fileStatusService.updateFileModificationDate(fileId, OffsetDateTime.now());

73
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java Normal file
View File

@ -0,0 +1,73 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.experimental.UtilityClass;
@UtilityClass
public class FileStatusMapper {
public FileStatus toFileStatus(FileModel status) {
return FileStatus.builder()
.dossierId(status.getDossierId())
.dossierArchived(status.isDossierArchived())
.dossierStatusId(status.getDossierStatusId())
.dossierTemplateId(status.getDossierTemplateId())
.fileId(status.getId())
.filename(status.getFilename())
.processingStatus(ProcessingStatus.valueOf(status.getProcessingStatus().name()))
.workflowStatus(WorkflowStatus.valueOf(status.getWorkflowStatus().name()))
.numberOfPages(status.getNumberOfPages())
.added(status.getAdded())
.lastUpdated(status.getLastUpdated())
.numberOfAnalyses(status.getNumberOfAnalyses())
.assignee(status.getAssignee())
.lastReviewer(status.getLastReviewer())
.lastApprover(status.getLastApprover())
.hasUpdates(status.isHasUpdates())
.hasImages(status.isHasImages())
.hasRequests(status.isHasSuggestions())
.hasSuggestions(status.isHasSuggestions())
.excludedFromAutomaticAnalysis(status.isExcludedFromAutomaticAnalysis())
.hasHints(status.isHasHints())
.hasRedactions(status.isHasRedactions())
.ocrEndTime(status.getOcrEndTime())
.ocrStartTime(status.getOcrStartTime())
.numberOfOCRedPages(status.getNumberOfOCRedPages() != null ? status.getNumberOfOCRedPages() : 0)
.numberOfPagesToOCR(status.getNumberOfPagesToOCR() != null ? status.getNumberOfPagesToOCR() : 0)
.hasAnnotationComments(status.isHasAnnotationComments())
.uploader(status.getUploader())
.dictionaryVersion(status.getDictionaryVersion())
.rulesVersion(status.getRulesVersion())
.componentRulesVersion(status.getComponentRulesVersion())
.legalBasisVersion(status.getLegalBasisVersion())
.lastProcessed(status.getLastProcessed())
.lastLayoutProcessed(status.getLastLayoutProcessed())
.approvalDate(status.getApprovalDate())
.lastUploaded(status.getLastUploaded())
.analysisDuration(status.getAnalysisDuration())
.fileAttributes(new FileAttributes(status.getFileAttributes()))
.dossierDictionaryVersion(status.getDossierDictionaryVersion())
.excluded(status.isExcluded())
.excludedPages(status.getExcludedPages())
.softDeletedTime(status.getDeleted())
.hardDeletedTime(status.getHardDeletedTime())
.analysisRequired(status.isAnalysisRequired())
.lastFileAttributeChange(status.getLastFileAttributeChange())
.redactionModificationDate(status.getRedactionModificationDate())
.fileManipulationDate(status.getFileManipulationDate())
.lastManualChangeDate(status.getLastManualChangeDate())
.hasHighlights(status.isHasHighlights())
.lastIndexed(status.getLastIndexed())
.fileSize(status.getFileSize())
.fileErrorInfo(status.getFileErrorInfo())
.build();
}
}

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/layoutparsing/LayoutParsingRequestFactory.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing;
import java.util.Map;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Value;

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/FilteredEntityLogRequest.java
View File

@ -15,8 +15,6 @@ import lombok.NoArgsConstructor;
public class FilteredEntityLogRequest {
private List<String> excludedTypes;
private boolean withManualRedactions;
private boolean includeFalsePositives;
private OffsetDateTime specifiedDate;
}

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/ManualChange.java
View File

@ -4,7 +4,6 @@ import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Map;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import lombok.AllArgsConstructor;
@ -18,7 +17,6 @@ import lombok.NoArgsConstructor;
@Builder
public class ManualChange {
private AnnotationStatus annotationStatus;
private ManualRedactionType manualRedactionType;
private OffsetDateTime processedDate;
private OffsetDateTime requestedDate;
@ -29,7 +27,6 @@ public class ManualChange {
public static ManualChange from(BaseAnnotation baseAnnotation) {
ManualChange manualChange = new ManualChange();
manualChange.annotationStatus = baseAnnotation.getStatus();
manualChange.processedDate = baseAnnotation.getProcessedDate();
manualChange.requestedDate = baseAnnotation.getRequestDate();
manualChange.userId = baseAnnotation.getUser();

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/Comment.java
View File

@ -2,8 +2,6 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.annotation
import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;