From 7bc374783cb52a3a9bc491563f0dbb50542a9ce8 Mon Sep 17 00:00:00 2001 From: aoezyetimoglu Date: Mon, 9 May 2022 17:34:29 +0200 Subject: [PATCH 1/5] RED-3985: No Re-Analyze after set Vertebrate Study to Yes --- .../FileStatusPersistenceService.java | 25 +++++++++++-------- .../controller/FileAttributesController.java | 1 - .../v1/server/service/FileStatusService.java | 7 ++++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 212372414..c0814670d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -1,5 +1,18 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.transaction.Transactional; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; @@ -9,18 +22,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Service; -import javax.transaction.Transactional; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java index a2c78a281..a2befdc2f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java @@ -124,7 +124,6 @@ public class FileAttributesController implements FileAttributesResource { } - @Transactional public void setFileAttributes(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Map fileAttributesMap) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index ff9d6a64d..b947fa633 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -220,7 +220,7 @@ public class FileStatusService { return; } - MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus()); + MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel); if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); @@ -262,7 +262,7 @@ public class FileStatusService { private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, - ProcessingStatus processingStatus) { + ProcessingStatus processingStatus, FileModel fileModel) { if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { return MessageType.STRUCTURE_ANALYSE; @@ -271,6 +271,9 @@ public class FileStatusService { return MessageType.ANALYSE; } if (reanalyse) { + if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) { + return MessageType.ANALYSE; + } return MessageType.REANALYSE; } return MessageType.ANALYSE; From 7d4da5fa8382e3805092d00413d272be98bd1db5 Mon Sep 17 00:00:00 2001 From: aoezyetimoglu Date: Wed, 11 May 2022 17:18:48 +0200 Subject: [PATCH 2/5] RED-4007: INC12878261: Cannot set dossier dictionary to "addToDictionaryAction": true --- .../service/persistence/DictionaryPersistenceService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java index 48ed03c7a..67208bb51 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java @@ -79,9 +79,11 @@ public class DictionaryPersistenceService { type.setRecommendationHexColor(typeValueRequest.getRecommendationHexColor()); type.setDescription(typeValueRequest.getDescription()); type.setLabel(typeValueRequest.getLabel()); + type.setAddToDictionaryAction(typeValueRequest.isAddToDictionaryAction()); } else { BeanUtils.copyProperties(typeValueRequest, type, "type", "dossierTemplateId", "dossierId", "entries", "falsePositiveEntries", "falseRecommendationEntries", "dossierTemplate", "dossier", "id", "version"); } + typeRepository.save(type); }); } From b5c6269e0936aa9448a4b6e002fce01ce9edd60d Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Fri, 13 May 2022 08:49:54 +0300 Subject: [PATCH 3/5] Fixed order of analysis RED-4017 --- .../peristence/v1/server/service/job/AutomaticAnalysisJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java index 88f30f0d7..7d52fd0db 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java @@ -44,7 +44,7 @@ public class AutomaticAnalysisJob implements Job { // queue up 5 files var allStatuses = getAllRelevantStatuses(); - allStatuses.sort(Collections.reverseOrder(Comparator.comparing(FileModel::getLastUpdated))); + allStatuses.sort(Comparator.comparing(FileModel::getLastUpdated)); var allStatusesIterator = allStatuses.iterator(); log.info("Files that require reanalysis: {}", allStatuses.size()); From 389bd6a35fb006bc93085cf5a5fa6c32b4cbf79e Mon Sep 17 00:00:00 2001 From: aoezyetimoglu Date: Tue, 10 May 2022 11:57:04 +0200 Subject: [PATCH 4/5] RED-3900: INC12775037 As a user I want to keep manual redactions if I re-upload a file --- .../v1/api/resources/UploadResource.java | 2 +- .../FileStatusPersistenceService.java | 20 ++-- .../repository/FileRepository.java | 11 +++ .../server/controller/UploadController.java | 5 +- .../v1/server/service/FileService.java | 4 +- .../v1/server/service/FileStatusService.java | 21 ++++- .../service/FileTesterAndProvider.java | 2 +- .../v1/server/integration/tests/FileTest.java | 92 +++++++++++++++---- 8 files changed, 121 insertions(+), 36 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java index 0982b26af..f94a7b672 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java @@ -29,7 +29,7 @@ public interface UploadResource { @PostMapping(value = UPLOAD_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - JSONPrimitive upload(@RequestBody AddFileRequest request); + JSONPrimitive upload(@RequestBody AddFileRequest request, @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions); @DeleteMapping(value = DELETE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index c0814670d..46dcb97cc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -300,14 +300,22 @@ public class FileStatusPersistenceService { @Transactional - public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights) { + public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) { - fileRepository.findById(fileId).ifPresent((file) -> { - file.setExcludedPages(new HashSet<>()); - }); + int countUpdate = 0; + + if(keepManualRedactions) { + countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); + } else { + + fileRepository.findById(fileId).ifPresent((file) -> { + file.setExcludedPages(new HashSet<>()); + }); + + countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); + } - int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 5a421ca87..56bd7331d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -133,6 +133,17 @@ public interface FileRepository extends JpaRepository { int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated, boolean hasHighlights); + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId") + int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded, + OffsetDateTime lastUpdated, boolean hasHighlights); + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java index 50f962848..9ca0c4cec 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java @@ -11,6 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.resources.UploadResource import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import javax.transaction.Transactional; @@ -29,9 +30,9 @@ public class UploadController implements UploadResource { private final DossierService dossierService; @Override - public JSONPrimitive upload(@RequestBody AddFileRequest request) { + public JSONPrimitive upload(@RequestBody AddFileRequest request, @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions) { - return fileService.upload(request); + return fileService.upload(request, keepManualRedactions); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 2129c2a45..3cfc8a2fd 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -49,7 +49,7 @@ public class FileService { private final IndexingService indexingService; - public JSONPrimitive upload(AddFileRequest request) { + public JSONPrimitive upload(AddFileRequest request, boolean keepManualRedactions) { dossierPersistenceService.getAndValidateDossier(request.getDossierId()); @@ -79,7 +79,7 @@ public class FileService { if (existingStatus != null) { // the file is already uploaded, just reanalyse it. - fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights()); + fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights(), keepManualRedactions); } else { // the file is new, should create a new status for it. log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index b947fa633..ac0406296 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -9,7 +9,6 @@ import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; @@ -436,17 +435,29 @@ public class FileStatusService { } - public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) { + public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) { + + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); + + if(keepManualRedactions) { + fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, true); + } else { + fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, false); + deleteManualRedactions(dossierId, fileId); + } - fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights); - wipeFileData(dossierId, fileId); viewedPagesPersistenceService.deleteForFile(fileId); + setStatusFullReprocess(dossierId, fileId, false, true); } @Transactional - public void wipeFileData(String dossierId, String fileId) { + public void deleteManualRedactions(String dossierId, String fileId) { OffsetDateTime now = OffsetDateTime.now(); // remove everything related to redaction diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index 3d694447d..f0bc71e01 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -71,7 +71,7 @@ public class FileTesterAndProvider { AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1"); fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, "test".getBytes(StandardCharsets.UTF_8)); - JSONPrimitive uploadResult = uploadClient.upload(upload); + JSONPrimitive uploadResult = uploadClient.upload(upload, false); return uploadResult.getValue(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 566883c44..a92bb3fa3 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -7,18 +7,28 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionGrid; + import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import java.time.OffsetDateTime; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; @@ -51,6 +61,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Autowired private ViewedPagesClient viewedPagesClient; + @Autowired + private RedactionLogClient redactionLogClient; + + @Test public void testFileSoftDeleteReupload() { @@ -77,6 +91,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getFilename()).isEqualTo(filename); } + @Test public void testFileOverwrite() { @@ -100,7 +115,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(viewedPages.size()).isEqualTo(1); AddFileRequest upload = new AddFileRequest(filename, file.getId(), dossier.getId(), "1"); - JSONPrimitive uploadResult = uploadClient.upload(upload); + JSONPrimitive uploadResult = uploadClient.upload(upload, false); loadedFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue()); assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isFalse(); @@ -109,8 +124,52 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(viewedPages.size()).isEqualTo(0); } + + @Test + public void testFileOverwriteAndKeepManualRedactions() { + + var filename = "test.pdf"; + var dossier = dossierTesterAndProvider.provideTestDossier(); + + var file = fileTesterAndProvider.testAndProvideFile(dossier, filename); + + fileClient.setStatusUnderReview(dossier.getId(), file.getId(), "1"); + fileClient.excludePages(dossier.getId(), file.getId(), Set.of(1)); + var redactionLog = redactionLogClient.getRedactionLog(dossier.getId(), file.getId(), null, false, true); + + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); + var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getFilename()).isEqualTo(filename); + + fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), "1"); + fileClient.toggleAutomaticAnalysis(dossier.getId(), file.getId(), true); + loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue(); + + viewedPagesClient.addPage(file.getId(), "1", 1); + var viewedPages = viewedPagesClient.getViewedPages(file.getId(), "1"); + + assertThat(viewedPages.size()).isEqualTo(1); + + AddFileRequest upload = new AddFileRequest(filename, file.getId(), dossier.getId(), "1"); + JSONPrimitive uploadResult = uploadClient.upload(upload, true); + + loadedFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue()); + assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue(); + + viewedPages = viewedPagesClient.getViewedPages(file.getId(), "1"); + assertThat(viewedPages.size()).isEqualTo(0); + + assertThat(loadedFile.getAssignee().equals("1")); + assertThat(loadedFile.getWorkflowStatus().equals(file.getWorkflowStatus())); + assertThat(loadedFile.getExcludedPages().equals(file.getExcludedPages())); + + } + + @Test public void testFile() { + var start = OffsetDateTime.now(); var dossier = dossierTesterAndProvider.provideTestDossier(); @@ -131,7 +190,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getFileManipulationDate()).isNotNull(); - fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), "1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getAssignee()).isEqualTo("1"); @@ -139,17 +197,14 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastReviewer()).isNull(); assertThat(loadedFile.getLastApprover()).isNull(); - fileClient.excludePages(dossier.getId(), file.getId(), Sets.newHashSet(1)); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getExcludedPages()).containsExactlyInAnyOrder(1); - fileClient.includePages(dossier.getId(), file.getId(), Sets.newHashSet(1)); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getExcludedPages()).isEmpty(); - fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), "1"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getAssignee()).isEqualTo("1"); @@ -157,7 +212,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastReviewer()).isNull(); assertThat(loadedFile.getLastApprover()).isNull(); - fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); @@ -180,7 +234,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastApprover()).isEqualTo("1"); assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue(); - fileClient.setStatusUnderReview(dossier.getId(), file.getId(), "2"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW); @@ -189,7 +242,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastApprover()).isEqualTo("1"); assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue(); - fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), "2"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL); @@ -197,7 +249,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastReviewer()).isEqualTo("2"); assertThat(loadedFile.getLastApprover()).isEqualTo("1"); - fileClient.setStatusApproved(dossier.getId(), file.getId(), "2"); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED); @@ -205,15 +256,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastReviewer()).isEqualTo("2"); assertThat(loadedFile.getLastApprover()).isEqualTo("2"); - fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null); - fileClient.toggleExclusion(dossier.getId(), file.getId(), true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.isExcluded()).isTrue(); - fileClient.toggleExclusion(dossier.getId(), file.getId(), false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.isExcluded()).isFalse(); @@ -225,14 +273,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var activeFiles = fileClient.getDossierStatus(dossier.getId()); assertThat(activeFiles.size()).isEqualTo(0); - uploadClient.undeleteFiles(dossier.getId(), Sets.newHashSet(file.getId())); softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()); assertThat(softDeletedFiles.size()).isEqualTo(0); activeFiles = fileClient.getDossierStatus(dossier.getId()); assertThat(activeFiles.size()).isEqualTo(1); - uploadClient.hardDeleteFiles(dossier.getId(), Sets.newHashSet(file.getId())); softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()); assertThat(softDeletedFiles.size()).isEqualTo(0); @@ -307,7 +353,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(manualRedactionClient.getLegalBasisChange(fileId, legalBasisChange.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId()); assertThat(manualRedactionClient.getImageRecategorization(fileId, imageRecategorization.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId()); - uploadClient.deleteFile(dossier.getId(), file.getId()); var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()); assertThat(softDeletedFiles.size()).isEqualTo(1); @@ -329,8 +374,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest { } + @Test public void testToggleEnableRedactionTwice() { + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); @@ -344,9 +391,17 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); - var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder().addToDictionary(true) - .addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId) - .user("user").reason("1").value("test").legalBasis("1").build())).iterator().next(); + var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder() + .addToDictionary(true) + .addToDossierDictionary(false) + .comment("comment") + .status(AnnotationStatus.REQUESTED) + .typeId(typeId) + .user("user") + .reason("1") + .value("test") + .legalBasis("1") + .build())).iterator().next(); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); @@ -357,7 +412,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.isExcluded()).isTrue(); assertThat(manualRedactionClient.getAddRedaction(fileId, addRedaction.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId()); - fileClient.toggleExclusion(dossier.getId(), file.getId(), false); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.isExcluded()).isFalse(); From 5cd160ffbb77a8825110e1f08d422d3a1a2d69ac Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Tue, 17 May 2022 13:15:00 +0200 Subject: [PATCH 5/5] RED-3554 Added flag to Resize endpoint to update dictionary --- .../annotations/ResizeRedactionRequest.java | 1 + .../ManualResizeRedactionEntity.java | 27 ++++++++---- .../service/ManualRedactionService.java | 42 ++++++++++++++++--- ...ictionary-to-manual-resize-redactions.yaml | 12 ++++++ .../db/changelog/db.changelog-master.yaml | 2 + 5 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ResizeRedactionRequest.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ResizeRedactionRequest.java index bde46246d..7fb4f90a2 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ResizeRedactionRequest.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ResizeRedactionRequest.java @@ -20,6 +20,7 @@ public class ResizeRedactionRequest { private String comment; private int page; private String value; + private Boolean updateDictionary; @Builder.Default private List positions = new ArrayList<>(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java index 9faafee68..45e5738c9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java @@ -1,18 +1,29 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotations; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.Column; +import javax.persistence.ElementCollection; +import javax.persistence.EmbeddedId; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; - -import javax.persistence.*; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; @Data @Builder @@ -39,6 +50,8 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation { private int page; @Column(length = 4000) private String value; + @Column + private Boolean updateDictionary; @ManyToOne(fetch = FetchType.LAZY) private FileEntity fileStatus; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index ff9716244..52cd24ee6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -7,6 +7,7 @@ import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -41,6 +42,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; @@ -96,11 +98,14 @@ public class ManualRedactionService { private final StopwordService stopwordService; private final RabbitTemplate rabbitTemplate; private final ObjectMapper objectMapper; + private final EntryPersistenceService entryPersistenceService; + private final RedactionLogService redactionLogService; private final HashFunction hashFunction = Hashing.murmur3_128(); public List addAddRedaction(String dossierId, String fileId, List addRedactionRequests) { + var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); @@ -468,11 +473,35 @@ public class ManualRedactionService { List resizeRedactionRequests) { var response = new ArrayList(); - dossierPersistenceService.getAndValidateDossier(dossierId); + var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - for (var resizeRedactionRequest : resizeRedactionRequests) { + boolean reprocess = false; + RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true, true); + + for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) { resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest); + if (resizeRedactionRequest.getUpdateDictionary() != null && resizeRedactionRequest.getUpdateDictionary()) { + + Optional redactionLogEntry = redactionLog.getRedactionLogEntry() + .stream() + .filter(r -> r.getId().equals(resizeRedactionRequest.getAnnotationId())) + .findFirst(); + if (redactionLogEntry.isPresent() && (redactionLogEntry.get().isDictionaryEntry() || redactionLogEntry.get().isDossierDictionaryEntry())) { + + String typeId = redactionLogEntry.get().getType() + ":" + dossier.getDossierTemplateId(); + + if (redactionLogEntry.get().getValue().length() > resizeRedactionRequest.getValue().length()) { + log.info("Remove old value '{}' from dictionary", redactionLogEntry.get().getValue()); + entryPersistenceService.deleteEntries(typeId, List.of(redactionLogEntry.get().getValue()), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY); + } + + log.info("Add new value '{}' to dictionary", resizeRedactionRequest.getValue()); + entryPersistenceService.addEntries(typeId, new HashSet<>(Collections.singleton(resizeRedactionRequest.getValue())), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY); + reprocess = true; + } + } + Long commentId = null; if (resizeRedactionRequest.getComment() != null) { @@ -482,10 +511,11 @@ public class ManualRedactionService { ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build(); addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - response.add(ManualAddResponse.builder() - .annotationId(resizeRedactionRequest.getAnnotationId()) - .commentId(commentId) - .build()); + response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); + } + + if (reprocess) { + reprocess(dossierId, fileId); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml new file mode 100644 index 000000000..7caffbcc6 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml @@ -0,0 +1,12 @@ +databaseChangeLog: + - changeSet: + id: add-update-dictionary-to-manual-resize-redactions + author: philipp + changes: + - addColumn: + columns: + - column: + name: update_dictionary + type: BOOLEAN + tableName: manual_resize_redaction + diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index a6ef2f15e..c496b232f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -67,3 +67,5 @@ databaseChangeLog: file: db/changelog/sql/26-initiliaze-application-configuration-table.sql - include: file: db/changelog/sql/27-update-soft-delete-date.sql + - include: + file: db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml