Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3720: Save annotations and TextHighlight PathElements on upload to TEXT_HIGHLIGHTS file

Browse Source
This commit is contained in:
deiflaender 2022-04-01 12:15:23 +02:00
parent 5df0ad68e3
commit e8afcc0afb
11 changed files with 81 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -61,6 +61,7 @@ public class FileModel {
private String dossierId;
private OffsetDateTime redactionModificationDate;
private OffsetDateTime fileManipulationDate;
private boolean hasHighlights;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;

5
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java
View File

@ -11,6 +11,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
@ -20,6 +21,7 @@ public interface ReanalysisResource {
String REANALYZE_PATH = "/reanalyze";
String IMPORT_REDACTIONS_PATH = "/import-redactions";
String TEXT_HIGHLIGHT_CONVERSION_PATH = "/texthighlights-conversion";
String CONVERT_TEXT_HIGHLIGHTS_PATH = "/convert-texthighlights";
String OCR_REANALYZE_PATH = "/ocr/reanalyze";
String REINDEX_PATH = "/reindex";
String DOSSIER_ID_PARAM = "dossierId";
@ -65,4 +67,7 @@ public interface ReanalysisResource {
@PostMapping(value = TEXT_HIGHLIGHT_CONVERSION_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest);
@PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest);
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -138,6 +138,9 @@ public class FileEntity {
@Column
private OffsetDateTime lastManualChangeDate;
@Column
private boolean hasHighlights;
@Column(columnDefinition = "text", name = "excluded_pages")
@Convert(converter = JSONIntegerSetConverter.class)
private Set<Integer> excludedPages = new HashSet<>();

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -31,7 +31,7 @@ public class FileStatusPersistenceService {
private final DossierPersistenceService dossierService;
public void createStatus(String dossierId, String fileId, String filename, String uploader) {
public void createStatus(String dossierId, String fileId, String filename, String uploader, boolean hasHighlights) {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
FileEntity file = new FileEntity();
@ -46,6 +46,7 @@ public class FileStatusPersistenceService {
file.setLastUploaded(now);
file.setLastUpdated(now);
file.setFileManipulationDate(now);
file.setHasHighlights(hasHighlights);
fileRepository.save(file);
}
@ -295,14 +296,14 @@ public class FileStatusPersistenceService {
@Transactional
public void overwriteFile(String fileId, String uploader, String filename) {
public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights) {
fileRepository.findById(fileId).ifPresent((file) -> {
file.setExcludedPages(new HashSet<>());
});
int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
if (countUpdate == 0) {
throw new NotFoundException("Unknown file=" + fileId);
}
@ -364,4 +365,9 @@ public class FileStatusPersistenceService {
fileRepository.updateFileModificationDate(fileId, fileManipulationDate);
}
@Transactional
public void updateHasHighlights(String fileId, boolean hasHighlights){
fileRepository.updateHasHighlights(fileId, hasHighlights);
}
}

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -78,6 +78,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.fileManipulationDate = :fileManipulationDate where f.id = :fileId")
void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate);
@Modifying
@Query("update FileEntity f set f.hasHighlights = :hasHighlights where f.id = :fileId")
void updateHasHighlights(String fileId, boolean hasHighlights);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.hasAnnotationComments = :hasAnnotationComments where f.id = :fileId")
void updateHasComments(String fileId, OffsetDateTime lastUpdated, boolean hasAnnotationComments);
@ -120,10 +124,10 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
"f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," +
"f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
"f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " +
"f.deleted = null, f.hardDeletedTime = null " +
"f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " +
"where f.id = :fileId")
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus,
WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated);
WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated, boolean hasHighlights);
@Query("select count(f) from FileEntity f where f.dossierId = :dossierId and f.deleted is not null and f.hardDeletedTime is null")
int countSoftDeletedFiles(String dossierId);

32
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -17,6 +17,8 @@ import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
@ -47,7 +49,8 @@ public class ReanalysisController implements ReanalysisResource {
@Override
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
var relevantFiles = getAllFilesForDossier(dossierId, true);
reanalyseFiles(dossierId, force, relevantFiles);
@ -84,7 +87,8 @@ public class ReanalysisController implements ReanalysisResource {
public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set<String> fileIds) {
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex,
@RequestBody Set<String> fileIds) {
indexingService.reindex(dossierId, fileIds, dropIndex);
}
@ -113,8 +117,11 @@ public class ReanalysisController implements ReanalysisResource {
var textHighlightResponse = pDFTronRedactionClient.processTextHighlights(textHighlightRequest);
if (textHighlightRequest.getOperation().equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)) {
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
if (textHighlightRequest.getOperation()
.equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation()
.equals(TextHighlightOperation.CONVERT)) {
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS));
}
if (textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)) {
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true);
@ -123,6 +130,20 @@ public class ReanalysisController implements ReanalysisResource {
}
public void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest) {
boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest);
fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights);
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS));
if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true);
}
}
public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
@ -132,7 +153,8 @@ public class ReanalysisController implements ReanalysisResource {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
} else {
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus()
.equals(WorkflowStatus.APPROVED)) {
throw new ConflictException("Cannot analyse a deleted/approved file");
}
if (dossierFile.getLastOCRTime() != null) {

9
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java
View File

@ -78,8 +78,11 @@ public class FileService {
}
}
boolean hasHighlights;
try {
pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData()));
hasHighlights = pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData()));
} catch (FeignException e) {
log.warn("Failed to optimize file: {}", request.getFilename(), e);
@ -89,11 +92,11 @@ public class FileService {
if (existingStatus != null) {
// the file is already uploaded, just reanalyse it.
fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename());
fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights);
} else {
// the file is new, should create a new status for it.
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename());
fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights);
}
return new JSONPrimitive<>(fileId);
}

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -308,9 +308,9 @@ public class FileStatusService {
@Transactional
public void createStatus(String dossierId, String fileId, String uploader, String filename) {
public void createStatus(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, hasHighlights);
addToAnalysisQueue(dossierId, fileId, false, Set.of());
}
@ -356,6 +356,11 @@ public class FileStatusService {
}
public void updateHasHighlights(String fileId, boolean hasHighlights){
fileStatusPersistenceService.updateHasHighlights(fileId, hasHighlights);
}
public void setAssignee(String fileId, String assignee) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
@ -443,9 +448,9 @@ public class FileStatusService {
@Transactional
public void overwriteFile(String dossierId, String fileId, String uploader, String filename) {
public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename);
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights);
wipeFileData(dossierId, fileId);
setStatusFullReprocess(dossierId, fileId, false);
}

11
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/19-added-has-highlights-to-file.changelog.yaml Normal file
View File

@ -0,0 +1,11 @@
databaseChangeLog:
- changeSet:
id: added-has-highlights-to-file
author: dom
changes:
- addColumn:
columns:
- column:
name: has_highlights
type: BOOLEAN
tableName: file

4
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -42,4 +42,6 @@ databaseChangeLog:
- include:
file: db/changelog/17-digital-signature-kms.changelog.yaml
- include:
file: db/changelog/18-add-migration-table.yaml
file: db/changelog/18-add-migration-table.yaml
- include:
file: db/changelog/19-added-has-highlights-to-file.changelog.yaml

2
persistence-service-v1/pom.xml
View File

@ -27,7 +27,7 @@
<properties>
<redaction-service.version>3.90.0</redaction-service.version>
<search-service.version>2.26.0</search-service.version>
<pdftron-redaction-service.version>3.53.0</pdftron-redaction-service.version>
<pdftron-redaction-service.version>3.59.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.19.0</redaction-report-service.version>
</properties>