Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #284: States update

Browse Source 
Merge in RED/persistence-service from states-update to master

* commit '86aed6b961be243d77d039832c6977677a310340':
  dossier status count
  dossier status count
This commit is contained in:
Timo Bejan 2022-03-11 11:57:35 +01:00
commit d14b283a6c
17 changed files with 97 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/Dossier.java
View File

@ -37,5 +37,6 @@ public class Dossier {
private OffsetDateTime archivedTime;
private String dossierTemplateId;
private String dossierStatusId;
private DossierVisibility visibility;
}

6
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierVisibility.java Normal file
View File

@ -0,0 +1,6 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier;
public enum DossierVisibility {
PRIVATE, PUBLIC;
}

2
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -29,11 +29,11 @@ public class FileModel {
private OffsetDateTime deleted;
private OffsetDateTime lastProcessed;
private OffsetDateTime lastIndexed;
private OffsetDateTime lastManualChangeDate;
private int numberOfAnalyses;
private String assignee;
private String lastReviewer;
private String lastApprover;
private OffsetDateTime lastManualRedaction;
private boolean hasRedactions;
private boolean hasHints;
private boolean hasSuggestions;

8
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierStatusResource.java
View File

@ -11,7 +11,6 @@ import java.util.List;
public interface DossierStatusResource {
String DOSSIER_STATUS_PATH = "/dossier-status";
String DOSSIER_TEMPLATE_PATH = "/dossier-template";
String DOSSIER_TEMPLATE_ID = "dossierTemplateId";
@ -22,6 +21,7 @@ public interface DossierStatusResource {
String DOSSIER_STATUS_REPLACE_ID = "replaceDossierStatusId";
@ResponseBody
@ResponseStatus(HttpStatus.ACCEPTED)
@PostMapping(value = DOSSIER_STATUS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
@ -32,6 +32,12 @@ public interface DossierStatusResource {
@GetMapping(value = DOSSIER_STATUS_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
List<DossierStatusInfo> getAllDossierStatusForTemplate(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = DOSSIER_STATUS_PATH + DOSSIER_TEMPLATE_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
List<DossierStatusInfo> getAllDossierStatuses(@RequestBody List<String> dossierTemplateId);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@GetMapping(value = DOSSIER_STATUS_PATH + DOSSIER_STATUS_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java
View File

@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownl
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierVisibility;
import lombok.*;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
@ -84,6 +85,10 @@ public class DossierEntity {
@Column
private OffsetDateTime dueDate;
@Column
@Enumerated(EnumType.STRING)
private DossierVisibility visibility;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
private DossierTemplateEntity dossierTemplate;

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -69,9 +69,6 @@ public class FileEntity {
@Column
private String lastApprover;
@Column
private OffsetDateTime lastManualRedaction;
@Column
private boolean hasRedactions;
@ -138,6 +135,9 @@ public class FileEntity {
@Column
private OffsetDateTime fileManipulationDate;
@Column
private OffsetDateTime lastManualChangeDate;
@Column(columnDefinition = "text", name = "excluded_pages")
@Convert(converter = JSONIntegerSetConverter.class)
private Set<Integer> excludedPages = new HashSet<>();

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierStatusPersistenceService.java
View File

@ -13,6 +13,7 @@ import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
@ -56,7 +57,11 @@ public class DossierStatusPersistenceService {
}
public List<DossierStatusInfo> getAllDossierStatusForTemplate(String dossierTemplateId) {
return dossierStatusRepository.getAllDossierStatusForDossierTemplate(dossierTemplateId);
return dossierStatusRepository.getAllDossierStatusForDossierTemplate(Collections.singletonList(dossierTemplateId));
}
public List<DossierStatusInfo> getAllDossierStatuses(List<String> dossierTemplateIds) {
return dossierStatusRepository.getAllDossierStatusForDossierTemplate(dossierTemplateIds);
}
public DossierStatusInfo getDossierStatusInfo(String dossierStatusId) {

28
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -139,16 +139,6 @@ public class FileStatusPersistenceService {
}
@Transactional
public void updateLastManualRedaction(String fileId, OffsetDateTime date) {
if (isFileDeleted(fileId)) {
return;
}
fileRepository.updateLastManualRedaction(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date);
}
@Transactional
public void setUpdateStatusIndexingSuccessful(String fileId) {
@ -181,17 +171,6 @@ public class FileStatusPersistenceService {
}
@Transactional
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, boolean hasSuggestions) {
if (isFileDeleted(fileId)) {
return;
}
fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions);
}
public FileEntity getStatus(String fileId) {
return fileRepository.findById(fileId).orElseThrow(() -> new NotFoundException("Unknown file=" + fileId));
@ -207,7 +186,6 @@ public class FileStatusPersistenceService {
return;
}
file.setLastManualRedaction(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setExcludedPages(excludedPages);
}, () -> {
@ -360,6 +338,12 @@ public class FileStatusPersistenceService {
fileRepository.setLastRedactionModificationDateForFile(fileId, changeDate);
}
@Transactional
public void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate) {
fileRepository.setLastManualChangeDate(fileId, lastManualChangeDate);
}
public boolean hasChangesSince(String dossierId, OffsetDateTime since) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierStatusRepository.java
View File

@ -11,8 +11,8 @@ import java.util.Optional;
public interface DossierStatusRepository extends JpaRepository<DossierStatusEntity, String> {
@Query("select new com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo(s.id, s.name," +
"s.description, s.color, s.dossierTemplateId, count(d)) from DossierStatusEntity s left outer join s.dossiers d where s.dossierTemplateId = :dossierTemplateId group by s")
List<DossierStatusInfo> getAllDossierStatusForDossierTemplate(String dossierTemplateId);
"s.description, s.color, s.dossierTemplateId, count(d)) from DossierStatusEntity s left outer join s.dossiers d where s.dossierTemplateId in ( :dossierTemplateIds ) group by s")
List<DossierStatusInfo> getAllDossierStatusForDossierTemplate(List<String> dossierTemplateIds);
@Query("select new com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatusInfo(s.id, s.name," +
"s.description, s.color, s.dossierTemplateId, count(d)) from DossierStatusEntity s left outer join s.dossiers d where s.id = :dossierStatusId group by s")

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -82,15 +82,6 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.hasAnnotationComments = :hasAnnotationComments where f.id = :fileId")
void updateHasComments(String fileId, OffsetDateTime lastUpdated, boolean hasAnnotationComments);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastManualRedaction = :lastManualRedaction where f.id = :fileId")
void updateLastManualRedaction(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastManualRedaction);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastManualRedaction = :lastManualRedaction, " +
"f.hasSuggestions = :hasSuggestions where f.id = :fileId")
void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastManualRedaction,
boolean hasSuggestions);
@Modifying
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " +
@ -126,7 +117,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
"f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " +
"f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " +
"f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, f.numberOfAnalyses = 0, " +
"f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," +
"f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
"f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " +
"f.deleted = null, f.hardDeletedTime = null " +
@ -144,6 +135,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.redactionModificationDate = :redactionModificationDate where f.id = :fileId")
void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime redactionModificationDate);
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.lastManualChangeDate = :lastManualChangeDate where f.id = :fileId")
void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate);
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " +
"and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'DELETED' or f.processingStatus = 'ERROR' )" +
"and d.softDeletedTime is null and d.hardDeletedTime is null and d.status = 'ACTIVE' " )

7
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierStatusController.java
View File

@ -7,6 +7,7 @@ import com.iqser.red.service.persistence.service.v1.api.resources.DossierStatusR
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@ -34,8 +35,12 @@ public class DossierStatusController implements DossierStatusResource {
}
@Override
public DossierStatusInfo getDossierStatus(@PathVariable(DOSSIER_STATUS_ID) String dossierStatusId) {
public List<DossierStatusInfo> getAllDossierStatuses(@RequestBody List<String> dossierTemplateIds) {
return convert(dossierStatusPersistenceService.getAllDossierStatuses(dossierTemplateIds), DossierStatusInfo.class);
}
@Override
public DossierStatusInfo getDossierStatus(@PathVariable(DOSSIER_STATUS_ID) String dossierStatusId) {
return dossierStatusPersistenceService.getDossierStatusInfo(dossierStatusId);
}

29
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java
View File

@ -1,13 +1,5 @@
package com.iqser.red.service.peristence.v1.server.service;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
@ -16,9 +8,15 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
@Service
@ -50,6 +48,7 @@ public class AnalysisFlagsCalculationService {
boolean hasComments = false;
OffsetDateTime lastModification = null;
OffsetDateTime lastManualChangeDate = null;
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
if (entry.isExcluded()) {
@ -64,10 +63,11 @@ public class AnalysisFlagsCalculationService {
for (var manualChange : entry.getManualChanges()) {
if ((manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.REMOVE_LOCALLY) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.FORCE_REDACT) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.RESIZE)) && manualChange.getProcessedDate()!= null && manualChange.getProcessedDate().isAfter(lastModification)) {
.equals(ManualRedactionType.FORCE_REDACT) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.RESIZE)) && manualChange.getProcessedDate() != null && manualChange.getProcessedDate().isAfter(lastModification)) {
lastModification = manualChange.getProcessedDate();
lastManualChangeDate = manualChange.getProcessedDate();
}
}
}
@ -116,6 +116,9 @@ public class AnalysisFlagsCalculationService {
if (lastModification != null && (file.getRedactionModificationDate() == null || file.getRedactionModificationDate().isBefore(lastModification))) {
fileStatusPersistenceService.setLastRedactionModificationDateForFile(fileId, lastModification);
}
if (lastManualChangeDate != null) {
fileStatusPersistenceService.setLastManualChangeDate(fileId, lastManualChangeDate);
}
}
@ -124,4 +127,4 @@ public class AnalysisFlagsCalculationService {
return typeId.split(":")[0];
}
}
}

15
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -255,7 +255,6 @@ public class ManualRedactionService {
var createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
fileStatusPersistenceService.updateHasComments(fileId, true);
return createdComment;
@ -387,7 +386,6 @@ public class ManualRedactionService {
public void deleteComment(String fileId, List<Long> commentIds) {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
for (var commentId : commentIds) {
commentPersistenceService.softDelete(commentId, OffsetDateTime.now());
}
@ -473,8 +471,6 @@ public class ManualRedactionService {
removeRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -486,8 +482,6 @@ public class ManualRedactionService {
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var annotationId : annotationIds) {
forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -506,8 +500,6 @@ public class ManualRedactionService {
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var annotationId : annotationIds) {
legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -526,8 +518,6 @@ public class ManualRedactionService {
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(annotationStatus);
recategorizationPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
}
if (actionPerformed) {
reprocess(dossierId, fileId);
@ -553,8 +543,6 @@ public class ManualRedactionService {
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var annotationId : annotationIds) {
resizeRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -583,9 +571,6 @@ public class ManualRedactionService {
addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS));
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasSuggestions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}

12
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/12-dossier-visibility.changelog.yaml Normal file
View File

@ -0,0 +1,12 @@
databaseChangeLog:
- changeSet:
id: add-dossier-visibility-column
author: timo
changes:
- addColumn:
columns:
- column:
name: visibility
type: VARCHAR(40)
defaultValue: PUBLIC
tableName: dossier

16
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/13-file-manual-change-date.changelog.yaml Normal file
View File

@ -0,0 +1,16 @@
databaseChangeLog:
- changeSet:
id: add-file-manual-change-date-column
author: timo
changes:
- removeColumn:
columns:
- name: last_manual_redaction
tableName: file
- addColumn:
columns:
- column:
name: last_manual_change_date
type: TIMESTAMP WITHOUT TIME ZONE
defaultValue: now()
tableName: file

6
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -24,4 +24,8 @@ databaseChangeLog:
- include:
file: db/changelog/sql/10-set-file-manipulation-date.sql
- include:
file: db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml
file: db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml
- include:
file: db/changelog/12-dossier-visibility.changelog.yaml
- include:
file: db/changelog/13-file-manual-change-date.changelog.yaml

1
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java
View File

@ -158,7 +158,6 @@ public class FilePerformanceTest extends AbstractPersistenceServerServiceTest {
f.setLastUploaded(OffsetDateTime.now());
f.setLastProcessed(OffsetDateTime.now());
f.setLastFileAttributeChange(OffsetDateTime.now());
f.setLastManualRedaction(OffsetDateTime.now());
f.setDossierId(d.getId());
files.add(f);