Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #121: Dossier stats updates

Browse Source 
Merge in RED/persistence-service from dossier-stats-updates to master

* commit '35d688c135442cb5cbc3806cea07368d107adae9':
  Test fix
  dossier information endpoint
  added counter for soft-deleted files in dossier-stats
This commit is contained in:
Timo Bejan 2021-11-22 09:35:43 +01:00
commit af5cb2d0e3
11 changed files with 98 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierInformation.java Normal file
View File

@ -0,0 +1,12 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier;
import lombok.Data;
@Data
public class DossierInformation {
private int numberOfActiveDossiers;
private int numberOfSoftDeletedDossiers;
private int numberOfHardDeletedDossiers;
}

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java
View File

@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
public class DossierStats {
private String dossierId;
private int numberOfFiles;
private int numberOfSoftDeletedFiles;
private int numberOfPages; // sum of pages
private int numberOfExcludedPages; // sum of excludedPages
private boolean hasRedactionsFilePresent; // true if at least one file in the dossier has redactions

7
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierResource.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.resources;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierInformation;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
@ -15,6 +16,7 @@ import java.util.Set;
public interface DossierResource {
String REST_PATH = "/dossier";
String INFO_PATH = "/info";
String DELETED_DOSSIERS_PATH = "/deletedDossiers";
String HARD_DELETE_PATH = "/hardDelete";
String UNDELETE_PATH = "/undelete";
@ -28,7 +30,7 @@ public interface DossierResource {
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = REST_PATH+CHANGES_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
@PostMapping(value = REST_PATH + CHANGES_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
JSONPrimitive<Boolean> hasChangesSince(@RequestBody JSONPrimitive<OffsetDateTime> since);
@PostMapping(value = REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@ -43,6 +45,9 @@ public interface DossierResource {
@GetMapping(value = REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
List<Dossier> getAllDossiers();
@GetMapping(value = REST_PATH + INFO_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
DossierInformation getDossierInformation();
@GetMapping(value = REST_PATH + DOSSIER_ID_PATH_PARAM, produces = MediaType.APPLICATION_JSON_VALUE)
Dossier getDossierById(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -138,5 +137,12 @@ public class FileEntity {
@Column(updatable = false, insertable = false, name = "dossier_id")
private String dossierId;
public boolean isSoftDeleted() {
return deleted != null;
}
public boolean isSoftOrHardDeleted() {
return deleted != null || hardDeletedTime != null || ProcessingStatus.DELETED.equals(processingStatus);
}
}

48
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -20,7 +20,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@Service
@ -106,8 +105,8 @@ public class FileStatusPersistenceService {
return;
}
fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED,
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
@ -159,9 +158,10 @@ public class FileStatusPersistenceService {
fileAttributesRepository.deleteByFileId(fileId);
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
if (file.isSoftOrHardDeleted()) {
return;
}
var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes);
file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
@ -176,9 +176,10 @@ public class FileStatusPersistenceService {
@Transactional
public void setExcludedPages(String fileId, Set<Integer> excludedPages) {
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
if (isFileDeleted(file)) {
if (file.isSoftOrHardDeleted()) {
return;
}
file.setLastManualRedaction(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setExcludedPages(excludedPages);
@ -197,7 +198,7 @@ public class FileStatusPersistenceService {
public List<FileEntity> getActiveFiles(String dossierId) {
return fileRepository.findByDossierId(dossierId).stream().filter(f -> !isFileDeleted(f)).collect(Collectors.toList());
return fileRepository.findByDossierId(dossierId).stream().filter(f -> !f.isSoftOrHardDeleted()).collect(Collectors.toList());
}
@ -217,7 +218,7 @@ public class FileStatusPersistenceService {
public void softDelete(String fileId, OffsetDateTime softDeletedTime) {
int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED,
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime.truncatedTo(ChronoUnit.MILLIS));
if (countUpdate == 0) {
throw new NotFoundException("Unknown file=" + fileId);
}
@ -260,8 +261,8 @@ public class FileStatusPersistenceService {
return;
}
int updateCount = fileRepository.setCurrentReviewer(fileId, currentReviewer, lastReviewer,
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
currentReviewer == null ? WorkflowStatus.UNASSIGNED : WorkflowStatus.UNDER_REVIEW);
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
currentReviewer == null ? WorkflowStatus.UNASSIGNED : WorkflowStatus.UNDER_REVIEW);
if (updateCount == 0) {
throw new NotFoundException("Unknown file=" + fileId);
}
@ -288,9 +289,9 @@ public class FileStatusPersistenceService {
});
int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS,
WorkflowStatus.UNASSIGNED,
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
WorkflowStatus.UNASSIGNED,
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
if (countUpdate == 0) {
throw new NotFoundException("Unknown file=" + fileId);
}
@ -317,28 +318,15 @@ public class FileStatusPersistenceService {
}
private boolean isFileDeleted(FileEntity file) {
if (file == null || file.getDeleted() != null || file.getHardDeletedTime() != null || ProcessingStatus.DELETED.equals(file.getProcessingStatus())) {
return true;
}
return false;
}
private boolean isFileDeleted(String fileId) {
AtomicBoolean isFileDeleted = new AtomicBoolean(false);
fileRepository.findById(fileId).ifPresentOrElse((file) -> {
isFileDeleted.set(isFileDeleted(file));
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
});
return isFileDeleted.get();
return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false);
}
public boolean hasChangesSince(String dossierId, OffsetDateTime since) {
return fileRepository.existsByDossierIdAndLastUpdatedIsAfter(dossierId, since);
}
public int countSoftDeletedFiles(String dossierId) {
return fileRepository.countSoftDeletedFiles(dossierId);
}
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -10,7 +10,6 @@ import org.springframework.data.jpa.repository.Query;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
public interface FileRepository extends JpaRepository<FileEntity, String> {
@ -117,6 +116,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
"where f.id = :fileId")
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus,
WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated);
@Query("select count(f) from FileEntity f where f.dossierId = :dossierId and f.deleted is not null and f.hardDeletedTime is null")
int countSoftDeletedFiles(String dossierId);
}

47
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java
View File

@ -1,20 +1,5 @@
package com.iqser.red.service.peristence.v1.server.controller;
import static com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException.DOSSIER_NOT_FOUND_MESSAGE;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.peristence.v1.server.service.DossierService;
import com.iqser.red.service.peristence.v1.server.service.FileService;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
@ -25,11 +10,24 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Dossi
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierInformation;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import com.iqser.red.service.persistence.service.v1.api.resources.DossierResource;
import feign.Param;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException.DOSSIER_NOT_FOUND_MESSAGE;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@RestController
@RequiredArgsConstructor
@ -94,6 +92,23 @@ public class DossierController implements DossierResource {
.collect(Collectors.toList()), Dossier.class, new DossierMapper());
}
@Override
public DossierInformation getDossierInformation() {
DossierInformation dossierInformation = new DossierInformation();
dossierService.getAllDossiers().forEach(d -> {
if (d.getHardDeletedTime() != null) {
dossierInformation.setNumberOfHardDeletedDossiers(dossierInformation.getNumberOfHardDeletedDossiers() + 1);
} else if (d.getSoftDeletedTime() != null) {
dossierInformation.setNumberOfSoftDeletedDossiers(dossierInformation.getNumberOfSoftDeletedDossiers() + 1);
} else {
dossierInformation.setNumberOfActiveDossiers(dossierInformation.getNumberOfActiveDossiers() + 1);
}
});
return dossierInformation;
}
@Override
@Transactional

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java
View File

@ -28,7 +28,9 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Set;
import static com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException.DOSSIER_NOT_FOUND_MESSAGE;
@ -52,6 +54,7 @@ public class UploadController implements UploadResource {
@Override
@Transactional
public void deleteFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
var dossier = dossierService.getDossierById(dossierId);
@ -59,7 +62,7 @@ public class UploadController implements UploadResource {
throw new DossierNotFoundException(String.format(DOSSIER_NOT_FOUND_MESSAGE, dossierId));
}
OffsetDateTime softDeleteTime = OffsetDateTime.now();
OffsetDateTime softDeleteTime = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileService.softDeleteFile(dossierId, fileId, softDeleteTime);
fileStatusService.setFileStatusDeleted(fileId, softDeleteTime);
}

1
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java
View File

@ -35,6 +35,7 @@ public class DossierStatsService {
// get the associated files
List<FileEntity> files = fileStatusService.getActiveFiles(dossierId);
dossierStats.setNumberOfFiles(files.size());
dossierStats.setNumberOfSoftDeletedFiles(fileStatusService.countSoftDeletedFiles(dossierId));
dossierStats.setNumberOfPages(files.stream().mapToInt(FileEntity::getNumberOfPages).sum());
dossierStats.setNumberOfExcludedPages(files.stream().mapToInt(f -> f.getExcludedPages().size()).sum());
files.stream().filter(FileEntity::isHasRedactions).findAny().ifPresent(

14
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -10,7 +10,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -227,7 +226,7 @@ public class FileStatusService {
}
var fileAttributes = fileStatus.getFileAttributes();
var analyseRequest = AnalyzeRequest.builder()
.dossierId(dossierId)
.sectionsToReanalyse(sectionsToReanalyse)
@ -280,7 +279,7 @@ public class FileStatusService {
public void overwriteFile(String dossierId, String fileId, String uploader, String filename, int length) {
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename);
wipeFileData(dossierId,fileId);
wipeFileData(dossierId, fileId);
setStatusFullReprocess(dossierId, fileId, length);
}
@ -291,8 +290,7 @@ public class FileStatusService {
}
public void wipeFileData(String dossierId, String fileId){
public void wipeFileData(String dossierId, String fileId) {
OffsetDateTime now = OffsetDateTime.now();
// remove everything related to redaction
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
@ -350,6 +348,10 @@ public class FileStatusService {
}
public boolean hasChangesSince(String dossierId, OffsetDateTime since) {
return fileStatusPersistenceService.hasChangesSince(dossierId,since);
return fileStatusPersistenceService.hasChangesSince(dossierId, since);
}
public int countSoftDeletedFiles(String dossierId) {
return fileStatusPersistenceService.countSoftDeletedFiles(dossierId);
}
}

6
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java
View File

@ -74,6 +74,12 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getSoftDeletedDossiers()).isEmpty();
assertThatThrownBy(() -> dossierClient.undeleteDossiers(Sets.newHashSet(dossier.getId()))).isInstanceOf(FeignException.class);
var dossierInformation = dossierClient.getDossierInformation();
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(1);
}
@Test