Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'master' into RED-3392

Browse Source
This commit is contained in:
Philipp Schramm 2022-02-24 08:36:46 +01:00
commit c60ee07cf1
12 changed files with 84 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -59,7 +59,8 @@ public class FileModel {
private Set<Integer> excludedPages = new HashSet<>();
private Map<String, String> fileAttributes = new HashMap<>();
private String dossierId;
private OffsetDateTime annotationModificationDate;
private OffsetDateTime redactionModificationDate;
private OffsetDateTime fileManipulationDate;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -133,7 +133,10 @@ public class FileEntity {
private int analysisVersion;
@Column
private OffsetDateTime annotationModificationDate;
private OffsetDateTime redactionModificationDate;
@Column
private OffsetDateTime fileManipulationDate;
@Column(columnDefinition = "text", name = "excluded_pages")
@Convert(converter = JSONIntegerSetConverter.class)

20
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -35,6 +35,7 @@ public class FileStatusPersistenceService {
public void createStatus(String dossierId, String fileId, String filename, String uploader) {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
FileEntity file = new FileEntity();
file.setId(fileId);
file.setDossierId(dossierId);
@ -42,10 +43,11 @@ public class FileStatusPersistenceService {
file.setProcessingStatus(ProcessingStatus.UNPROCESSED);
file.setWorkflowStatus(WorkflowStatus.NEW);
file.setNumberOfPages(0);
file.setAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setAdded(now);
file.setUploader(uploader);
file.setLastUploaded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastUploaded(now);
file.setLastUpdated(now);
file.setFileManipulationDate(now);
fileRepository.save(file);
}
@ -123,6 +125,7 @@ public class FileStatusPersistenceService {
return;
}
fileRepository.updateLastOCRTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time);
fileRepository.updateFileModificationDate(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
@ -347,9 +350,9 @@ public class FileStatusPersistenceService {
@Transactional
public void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime changeDate) {
public void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime changeDate) {
fileRepository.setLastAnnotationModificationDateForFile(fileId, changeDate);
fileRepository.setLastRedactionModificationDateForFile(fileId, changeDate);
}
@ -367,4 +370,11 @@ public class FileStatusPersistenceService {
public List<FileEntity> getAllRelevantStatusesForReanalysisScheduler() {
return fileRepository.getAllRelevantStatusesForReanalysisScheduler();
}
@Transactional
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate){
fileRepository.updateFileModificationDate(fileId, fileManipulationDate);
}
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -74,6 +74,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastOCRTime = :lastOCRTime where f.id = :fileId")
void updateLastOCRTime(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastOCRTime);
@Modifying
@Query("update FileEntity f set f.fileManipulationDate = :fileManipulationDate where f.id = :fileId")
void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.hasAnnotationComments = :hasAnnotationComments where f.id = :fileId")
void updateHasComments(String fileId, OffsetDateTime lastUpdated, boolean hasAnnotationComments);
@ -120,7 +124,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
"f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " +
"f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " +
"f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " +
"f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, f.numberOfAnalyses = 0, " +
"f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
@ -137,8 +141,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
List<String> findDossierChangeByLastUpdatedIsAfter(OffsetDateTime since);
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.annotationModificationDate = :annotationModificationDate where f.id = :fileId")
void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime annotationModificationDate);
@Query("update FileEntity f set f.redactionModificationDate = :redactionModificationDate where f.id = :fileId")
void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime redactionModificationDate);
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " +
"and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'DELETED' or f.processingStatus = 'ERROR' )" +
@ -148,6 +152,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.lastFileAttributeChange = :date, f.lastUpdated = :date")
void updateLastAttributeChangeDate(OffsetDateTime date);
}

6
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -25,6 +25,8 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -91,6 +93,10 @@ public class ReanalysisController implements ReanalysisResource {
public TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest){
var textHighlightResponse = pDFTronRedactionClient.processTextHighlights(textHighlightRequest);
if(textHighlightRequest.getOperation().equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)){
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
if(textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)){
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), 1);
}

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java
View File

@ -4,6 +4,7 @@ import java.time.OffsetDateTime;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
@ -13,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import lombok.RequiredArgsConstructor;
@ -58,7 +60,19 @@ public class AnalysisFlagsCalculationService {
var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges().get(entry.getChanges().size() - 1);
if (lastChange != null && (lastModification == null || lastChange.getDateTime().isAfter(lastModification))) {
if (entry.getManualChanges() != null && !entry.getManualChanges().isEmpty() && !entry.isHint() && !entry.isRecommendation() && StringUtils.isNotEmpty(entry.getReason())) {
for (var manualChange : entry.getManualChanges()) {
if ((manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.REMOVE_LOCALLY) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.FORCE_REDACT) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || manualChange.getManualRedactionType()
.equals(ManualRedactionType.RESIZE)) && manualChange.getProcessedDate().isAfter(lastModification)) {
lastModification = manualChange.getProcessedDate();
}
}
}
if (lastChange != null && (lastModification == null || lastChange.getDateTime().isAfter(lastModification)) && !entry.isHint() && !entry.isRecommendation()) {
lastModification = lastChange.getDateTime();
}
@ -99,13 +113,8 @@ public class AnalysisFlagsCalculationService {
fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates);
}
OffsetDateTime lastManualRedactionTime = file.getLastManualRedaction();
if (lastModification == null || lastManualRedactionTime != null && lastManualRedactionTime.isAfter(lastModification)) {
lastModification = lastManualRedactionTime;
}
if (lastModification != null && (file.getAnnotationModificationDate() == null || file.getAnnotationModificationDate().isBefore(lastModification))) {
fileStatusPersistenceService.setLastAnnotationModificationDateForFile(fileId, lastModification);
if (lastModification != null && (file.getRedactionModificationDate() == null || file.getRedactionModificationDate().isBefore(lastModification))) {
fileStatusPersistenceService.setLastRedactionModificationDateForFile(fileId, lastModification);
}
}
@ -115,4 +124,4 @@ public class AnalysisFlagsCalculationService {
return typeId.split(":")[0];
}
}
}

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -325,6 +325,11 @@ public class FileStatusService {
}
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate){
fileStatusPersistenceService.updateFileModificationDate(fileId, fileManipulationDate);
}
public void setAssignee(String dossierId, String fileId, String assignee) {
if (StringUtils.isNotEmpty(assignee) && !dossierPersistenceService.getAndValidateDossier(dossierId)

11
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/10-added-file-manipulation-date.changelog.yaml Normal file
View File

@ -0,0 +1,11 @@
databaseChangeLog:
- changeSet:
id: added-file-manipulation-date.changelog
author: dom
changes:
- addColumn:
columns:
- column:
name: file_manipulation_date
type: TIMESTAMP WITHOUT TIME ZONE
tableName: file

9
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/9-changed-annotation-modification-date.changelog.yaml Normal file
View File

@ -0,0 +1,9 @@
databaseChangeLog:
- changeSet:
id: changed-annotation-modification-date
author: ali
changes:
- renameColumn:
newColumnName: redaction_modification_date
oldColumnName: annotation_modification_date
tableName: file

4
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -17,3 +17,7 @@ databaseChangeLog:
file: db/changelog/sql/7.1-set-json-fields.sql
- include:
file: db/changelog/sql/7.2-set-dossier-status.sql
- include:
file: db/changelog/9-changed-annotation-modification-date.changelog.yaml
- include:
file: db/changelog/10-added-file-manipulation-date.changelog.yaml

1
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -120,6 +120,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getFileManipulationDate()).isNotNull();
fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), "1");

2
persistence-service-v1/pom.xml
View File

@ -27,7 +27,7 @@
<properties>
<redaction-service.version>3.76.0</redaction-service.version>
<search-service.version>2.18.0</search-service.version>
<pdftron-redaction-service.version>3.42.0</pdftron-redaction-service.version>
<pdftron-redaction-service.version>3.44.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.19.0</redaction-report-service.version>
</properties>