diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java index 9413fb765..fdc9837f0 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java @@ -61,6 +61,7 @@ public class FileModel { private String dossierId; private OffsetDateTime redactionModificationDate; private OffsetDateTime fileManipulationDate; + private boolean hasHighlights; public boolean isAnalysisRequired(){ return this.fullAnalysisRequired || this.reanalysisRequired; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java index 1d0bf3c8a..0121be7d1 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java @@ -11,6 +11,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseStatus; import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse; @@ -20,6 +21,7 @@ public interface ReanalysisResource { String REANALYZE_PATH = "/reanalyze"; String IMPORT_REDACTIONS_PATH = "/import-redactions"; String TEXT_HIGHLIGHT_CONVERSION_PATH = "/texthighlights-conversion"; + String CONVERT_TEXT_HIGHLIGHTS_PATH = "/convert-texthighlights"; String OCR_REANALYZE_PATH = "/ocr/reanalyze"; String REINDEX_PATH = "/reindex"; String DOSSIER_ID_PARAM = "dossierId"; @@ -65,4 +67,7 @@ public interface ReanalysisResource { @PostMapping(value = TEXT_HIGHLIGHT_CONVERSION_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest); + @PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE) + void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 3b375f1cd..4c4738e95 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -138,6 +138,9 @@ public class FileEntity { @Column private OffsetDateTime lastManualChangeDate; + @Column + private boolean hasHighlights; + @Column(columnDefinition = "text", name = "excluded_pages") @Convert(converter = JSONIntegerSetConverter.class) private Set excludedPages = new HashSet<>(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 11ea72a2d..9164ea992 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -31,7 +31,7 @@ public class FileStatusPersistenceService { private final DossierPersistenceService dossierService; - public void createStatus(String dossierId, String fileId, String filename, String uploader) { + public void createStatus(String dossierId, String fileId, String filename, String uploader, boolean hasHighlights) { OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); FileEntity file = new FileEntity(); @@ -46,6 +46,7 @@ public class FileStatusPersistenceService { file.setLastUploaded(now); file.setLastUpdated(now); file.setFileManipulationDate(now); + file.setHasHighlights(hasHighlights); fileRepository.save(file); } @@ -295,14 +296,14 @@ public class FileStatusPersistenceService { @Transactional - public void overwriteFile(String fileId, String uploader, String filename) { + public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights) { fileRepository.findById(fileId).ifPresent((file) -> { file.setExcludedPages(new HashSet<>()); }); int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -364,4 +365,9 @@ public class FileStatusPersistenceService { fileRepository.updateFileModificationDate(fileId, fileManipulationDate); } + @Transactional + public void updateHasHighlights(String fileId, boolean hasHighlights){ + fileRepository.updateHasHighlights(fileId, hasHighlights); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index e4bcb80f3..aeb952c35 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -78,6 +78,10 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.fileManipulationDate = :fileManipulationDate where f.id = :fileId") void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate); + @Modifying + @Query("update FileEntity f set f.hasHighlights = :hasHighlights where f.id = :fileId") + void updateHasHighlights(String fileId, boolean hasHighlights); + @Modifying @Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.hasAnnotationComments = :hasAnnotationComments where f.id = :fileId") void updateHasComments(String fileId, OffsetDateTime lastUpdated, boolean hasAnnotationComments); @@ -120,10 +124,10 @@ public interface FileRepository extends JpaRepository { "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + - "f.deleted = null, f.hardDeletedTime = null " + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId") int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, - WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated); + WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated, boolean hasHighlights); @Query("select count(f) from FileEntity f where f.dossierId = :dossierId and f.deleted is not null and f.hardDeletedTime is null") int countSoftDeletedFiles(String dossierId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index b8cb40903..d1576dbaa 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -17,6 +17,8 @@ import com.google.common.collect.Sets; import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse; +import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionOperation; +import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightOperation; import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest; import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse; @@ -47,7 +49,8 @@ public class ReanalysisController implements ReanalysisResource { @Override - public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { + public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { var relevantFiles = getAllFilesForDossier(dossierId, true); reanalyseFiles(dossierId, force, relevantFiles); @@ -84,7 +87,8 @@ public class ReanalysisController implements ReanalysisResource { public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId, - @RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set fileIds) { + @RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, + @RequestBody Set fileIds) { indexingService.reindex(dossierId, fileIds, dropIndex); } @@ -113,8 +117,11 @@ public class ReanalysisController implements ReanalysisResource { var textHighlightResponse = pDFTronRedactionClient.processTextHighlights(textHighlightRequest); - if (textHighlightRequest.getOperation().equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)) { - fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + if (textHighlightRequest.getOperation() + .equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation() + .equals(TextHighlightOperation.CONVERT)) { + fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS)); } if (textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)) { fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true); @@ -123,6 +130,20 @@ public class ReanalysisController implements ReanalysisResource { } + public void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest) { + + boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest); + + fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights); + fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS)); + + if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) { + fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true); + } + } + + public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { @@ -132,7 +153,8 @@ public class ReanalysisController implements ReanalysisResource { fileStatusService.setStatusOcrProcessing(dossierId, fileId); } else { FileModel dossierFile = fileStatusService.getStatus(fileId); - if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { + if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus() + .equals(WorkflowStatus.APPROVED)) { throw new ConflictException("Cannot analyse a deleted/approved file"); } if (dossierFile.getLastOCRTime() != null) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 6b5b703c7..dd03289e8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -78,8 +78,11 @@ public class FileService { } } + boolean hasHighlights; try { - pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData())); + hasHighlights = pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData())); + + } catch (FeignException e) { log.warn("Failed to optimize file: {}", request.getFilename(), e); @@ -89,11 +92,11 @@ public class FileService { if (existingStatus != null) { // the file is already uploaded, just reanalyse it. - fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename()); + fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights); } else { // the file is new, should create a new status for it. log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename()); - fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename()); + fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights); } return new JSONPrimitive<>(fileId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index f799cc8a7..879d3cc40 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -308,9 +308,9 @@ public class FileStatusService { @Transactional - public void createStatus(String dossierId, String fileId, String uploader, String filename) { + public void createStatus(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) { - fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader); + fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, hasHighlights); addToAnalysisQueue(dossierId, fileId, false, Set.of()); } @@ -356,6 +356,11 @@ public class FileStatusService { } + public void updateHasHighlights(String fileId, boolean hasHighlights){ + fileStatusPersistenceService.updateHasHighlights(fileId, hasHighlights); + } + + public void setAssignee(String fileId, String assignee) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -443,9 +448,9 @@ public class FileStatusService { @Transactional - public void overwriteFile(String dossierId, String fileId, String uploader, String filename) { + public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) { - fileStatusPersistenceService.overwriteFile(fileId, uploader, filename); + fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights); wipeFileData(dossierId, fileId); setStatusFullReprocess(dossierId, fileId, false); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/19-added-has-highlights-to-file.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/19-added-has-highlights-to-file.changelog.yaml new file mode 100644 index 000000000..c07e80fa8 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/19-added-has-highlights-to-file.changelog.yaml @@ -0,0 +1,11 @@ +databaseChangeLog: + - changeSet: + id: added-has-highlights-to-file + author: dom + changes: + - addColumn: + columns: + - column: + name: has_highlights + type: BOOLEAN + tableName: file diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 08a3de2b9..d2d98bc53 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -42,4 +42,6 @@ databaseChangeLog: - include: file: db/changelog/17-digital-signature-kms.changelog.yaml - include: - file: db/changelog/18-add-migration-table.yaml \ No newline at end of file + file: db/changelog/18-add-migration-table.yaml + - include: + file: db/changelog/19-added-has-highlights-to-file.changelog.yaml \ No newline at end of file diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index 18045b65b..593f087eb 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -27,7 +27,7 @@ 3.90.0 2.26.0 - 3.53.0 + 3.59.0 3.19.0