diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java index fd1cf97ac..81642b6ac 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java @@ -19,10 +19,10 @@ public interface DossierTemplateStatsResource { String DOSSIER_TEMPLATE_ID = "dossierTemplateId"; String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID + "}"; - @PostMapping(value = REST_PATH+DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + @PostMapping(value = REST_PATH + DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) List getDossierTemplateDictionaryStats(@RequestBody Set dossierTemplateIds); - @PostMapping(value = REST_PATH+DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE ) + @PostMapping(value = REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) DossierTemplateStats getDossierTemplateStats(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId); @PostMapping(value = REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java index 0defeeea8..f74a1767b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java @@ -113,4 +113,7 @@ public class DossierEntity { @Column(updatable = false, insertable = false, name = "dossier_status_id") private String dossierStatusId; + public boolean isDeleted() { + return softDeletedTime != null || hardDeletedTime != null; + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java index 87e2b1956..0870ea4da 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java @@ -29,20 +29,17 @@ public class MigrationPersistenceService { var migrations = migrationRepository.findByVersionGreaterThan(lastProcessedVersion); if (migrations == null || migrations.isEmpty()) { - if(version > lastProcessedVersion){ - return false; - } - return true; + return version <= lastProcessedVersion; } - return migrations.stream().filter(m -> m.getVersion() == version).findFirst().isPresent(); + return migrations.stream().anyMatch(m -> m.getVersion() == version); } public Long getLatestProcessedVersion() { var migrations = migrationRepository.findAll(); - if (migrations == null || migrations.isEmpty()) { + if (migrations.isEmpty()) { return null; } return migrations.stream().sorted(Comparator.comparing(MigrationEntity::getVersion).reversed()).collect(Collectors.toList()).get(0).getVersion(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java index 5319dfe49..38d631b2b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java @@ -1,8 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; @@ -20,7 +18,6 @@ public interface TypeRepository extends JpaRepository { List findByDossierId(String dossierId); @Query("select t from TypeEntity t where t.id = :typeId and t.softDeletedTime is null") - Optional findByIdAndNotDeleted(String typeId); @Modifying diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java index c573c983f..2013a378d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java @@ -1,25 +1,5 @@ package com.iqser.red.service.peristence.v1.server.controller; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.transaction.Transactional; - -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities; import com.iqser.red.service.peristence.v1.server.service.StopwordService; import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator; @@ -36,9 +16,25 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.transaction.Transactional; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; @RestController @RequiredArgsConstructor @@ -80,8 +76,8 @@ public class DictionaryController implements DictionaryResource { var currentVersion = getCurrentVersion(typeResult); if (removeCurrent) { - entryPersistenceService.deleteAllEntriesForTypeId(typeId,currentVersion+1,dictionaryEntryType); - entryPersistenceService.addEntries(typeId,cleanEntries, currentVersion+1, dictionaryEntryType); + entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType); + entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); } else { entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); } @@ -136,9 +132,9 @@ public class DictionaryController implements DictionaryResource { var currentVersion = getCurrentVersion(typeResult); - entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.ENTRY); + entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY); entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); - entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); } @@ -160,7 +156,7 @@ public class DictionaryController implements DictionaryResource { checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest .getLabel()); - if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(),false) + if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false) .stream() .anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null && typeRequest .getDossierId() @@ -203,7 +199,7 @@ public class DictionaryController implements DictionaryResource { @PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestParam(value = INCLUDE_DELETED_PARAMETER_NAME, required = false, defaultValue = "false") boolean includeDeleted) { - return convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId,includeDeleted), Type.class); + return convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, includeDeleted), Type.class); } @@ -211,7 +207,7 @@ public class DictionaryController implements DictionaryResource { public List getAllTypesForDossier(@PathVariable(DOSSIER_ID_PARAMETER_NAME) String dossierId, @RequestParam(value = INCLUDE_DELETED_PARAMETER_NAME, required = false, defaultValue = "false") boolean includeDeleted) { - return convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId,includeDeleted), Type.class); + return convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId, includeDeleted), Type.class); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index a4e8a4ed9..8f2a87e5f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -197,11 +197,11 @@ public class ReanalysisController implements ReanalysisResource { if (force) { filesToReanalyse.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true); + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); }); } else { filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true); + fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true); }); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java new file mode 100644 index 000000000..e7dea6f76 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java @@ -0,0 +1,151 @@ +package com.iqser.red.service.peristence.v1.server.internal; + + +import com.iqser.red.service.peristence.v1.server.service.DossierService; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.peristence.v1.server.service.FileStatusService; +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.core.AmqpAdmin; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.stream.Collectors; + +@Slf4j +@RestController +@RequiredArgsConstructor +@RequestMapping("/private/admin") +public class AdminInterfaceController { + + private final FileManagementStorageService fileManagementStorageService; + private final FileStatusService fileStatusService; + private final FileManagementServiceSettings fileManagementServiceSettings; + private final DossierService dossierService; + private final AmqpAdmin amqpAdmin; + + + + @PostMapping("/reset-file") + public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); + + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + + + } + + + @PostMapping("/reset-image-info") + public void resetImageInfo(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + if (fileManagementServiceSettings.isImageServiceEnabled()) { + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); + fileStatusService.addToImageQueue(dossierId, fileId); + } + } + + + @PostMapping("/ocr") + public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { + + fileStatusService.updateLastOCRTime(fileId); + fileStatusService.setStatusOcrProcessing(dossierId, fileId); + + } + + + @PostMapping("/ocr-all-files-that-have-already-been-ocrd") + public void forceOCR(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) { + + if (dryRun) { + log.info("Dry run!"); + } + + var dossiers = dossierService.getAllDossiers(); + for (var dossier : dossiers) { + if (!dossier.isDeleted()) { + var files = fileStatusService.getActiveFiles(dossier.getId()); + var filesThatRequireOCR = files.stream() + .filter(f -> !f.isExcluded()) + .filter(f -> !f.isExcludedFromAutomaticAnalysis()) + .filter(f -> !f.isSoftOrHardDeleted()) + .filter(f -> f.getLastOCRTime() != null) + .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList()); + + + for (var file : filesThatRequireOCR) { + log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}", + file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime()); + + if (!dryRun) { + fileStatusService.updateLastOCRTime(file.getId()); + fileStatusService.setStatusOcrProcessing(file.getDossierId(), file.getId()); + } + + } + + + } + } + } + + + @PostMapping("/reset-text-for-all-files") + public void resetText(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) { + + if (dryRun) { + log.info("Dry run!"); + } + + var dossiers = dossierService.getAllDossiers(); + for (var dossier : dossiers) { + if (!dossier.isDeleted()) { + var files = fileStatusService.getActiveFiles(dossier.getId()); + var filesThatRequireTextReset = files.stream() + .filter(f -> !f.isExcluded()) + .filter(f -> !f.isExcludedFromAutomaticAnalysis()) + .filter(f -> !f.isSoftOrHardDeleted()) + .filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList()); + + + for (var file : filesThatRequireTextReset) { + log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}", + file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime()); + + if (!dryRun) { + + var dossierId = file.getDossierId(); + var fileId = file.getId(); + + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); + fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); + + fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); + } + + } + + + } + } + } + + @PostMapping("/flush-queue") + public void resetText(@RequestParam() String queueName ) { + amqpAdmin.purgeQueue(queueName); + } + + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java index 90ec3e2df..540bfbbf0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java @@ -1,13 +1,11 @@ package com.iqser.red.service.peristence.v1.server.migration; -import org.springframework.beans.factory.annotation.Autowired; - import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService; - import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; @Data @Slf4j @@ -21,11 +19,14 @@ public abstract class Migration { @Setter private MigrationPersistenceService migrationPersistenceService; - public void run() { + run(false); + } + + public void run(boolean force) { var latestProcessedVersion = migrationPersistenceService.getLatestProcessedVersion(); - if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion)) { + if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion) || force) { log.info("Starting migration with name {} and version {}", name, version); migrate(); migrationPersistenceService.insertMigration(name, version); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java new file mode 100644 index 000000000..caef196df --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.peristence.v1.server.migration; + + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@Data +@RestController +@RequiredArgsConstructor +@RequestMapping("/private/migration") +public class MigrationController { + + private final List migrations; + + + @PostMapping + public void run(@RequestParam("migrationName") String migrationName, @RequestParam("force") boolean force) { + migrations.stream().filter(m -> m.getName().equalsIgnoreCase(migrationName) || m.getClass().getSimpleName().equalsIgnoreCase(migrationName)) + .forEach(m -> m.run(force)); + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java index dd0f8e12a..062f3001f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java @@ -1,20 +1,18 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.peristence.v1.server.service.ManualRedactionProviderService; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; @Slf4j @Setter diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java index 93b8ecf16..ec2a49bf7 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java @@ -1,10 +1,5 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import java.util.ArrayList; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.migration.Migration; @@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; @Slf4j @Setter @@ -56,8 +53,9 @@ public class DictionaryToEntityMigration2 extends Migration { files.forEach(file -> { log.info("Start migration of file {}", file.getId()); if (file.getHardDeletedTime() == null) { - var newRedactionLogEntries = new ArrayList(); + try { + var newRedactionLogEntries = new ArrayList(); var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId()); redactionLog.getRedactionLogEntry().forEach(entry -> { @@ -69,20 +67,21 @@ public class DictionaryToEntityMigration2 extends Migration { if (entry.getType().startsWith("recommendation_")) { entry.setType(entry.getType().substring(15)); entry.setRecommendation(true); - log.info("removed _recommendation"); + log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId()); } newRedactionLogEntries.add(entry); }); - redactionLog.setRedactionLogEntry(newRedactionLogEntries); - fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); - log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) { + redactionLog.setRedactionLogEntry(newRedactionLogEntries); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog)); + log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId()); + } } catch (JsonProcessingException e) { throw new RuntimeException("Migration failed"); } catch (NotFoundException e) { log.info("redactionLog {} does not exsist", file.getId()); - return; } } }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/EntityTypesMigration4.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/EntityTypesMigration4.java index 5f7668180..2d1e3ce8a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/EntityTypesMigration4.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/EntityTypesMigration4.java @@ -1,20 +1,18 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import java.util.List; -import java.util.Set; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Set; @Slf4j @Setter diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java index b547da4fb..560e1d8e6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java @@ -1,13 +1,11 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.peristence.v1.server.service.IndexingService; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; @Slf4j @Setter @@ -16,6 +14,8 @@ public class IndexMigration1 extends Migration { private static final String NAME = "Recreate index and index all files"; private static final long VERSION = 1; + @Autowired + private IndexingService indexingService; public IndexMigration1() { @@ -23,11 +23,6 @@ public class IndexMigration1 extends Migration { super(NAME, VERSION); } - - @Autowired - private IndexingService indexingService; - - @Override protected void migrate() { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java index d0cbabf78..5a90c2a2f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java @@ -32,7 +32,6 @@ public class MigrateHighlights3 extends Migration { @Autowired private FileManagementStorageService fileManagementStorageService; - public MigrateHighlights3() { super(NAME, VERSION); @@ -48,24 +47,32 @@ public class MigrateHighlights3 extends Migration { files.forEach(file -> { if (file.getHardDeletedTime() == null) { - var untouchedExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.UNTOUCHED); + try { + if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) { + var untouchedExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.UNTOUCHED); - if (!untouchedExists) { - var originExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.ORIGIN); + if (!untouchedExists) { + var originExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN); - if (!originExists) { - log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId()); - return; + if (!originExists) { + log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId()); + return; + } + + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED, + fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)); + + } + var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder() + .fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build()); + + fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights()); + } else { + log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); } - - fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED, - fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)); - + } catch (Exception e) { + log.warn("Failed to extract text highlights for document: {}", file.getId()); } - var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder() - .fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build()); - - fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights()); } }); }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java index 62bdeff36..eed84cff6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java @@ -1,18 +1,16 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import java.util.ArrayList; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import com.iqser.red.service.peristence.v1.server.controller.ManualRedactionController; import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; - import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; @Slf4j @Setter @@ -59,6 +57,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration { } }); } + + log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(), annotationIdsToRemove); + if (!annotationIdsToRemove.isEmpty()) { annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id)); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java index 81891338f..e28283485 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java @@ -1,5 +1,17 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; +import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; +import com.iqser.red.service.peristence.v1.server.migration.Migration; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.HashSet; import java.util.Locale; @@ -7,20 +19,6 @@ import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.peristence.v1.server.controller.DictionaryController; -import com.iqser.red.service.peristence.v1.server.migration.Migration; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; - -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - @Slf4j @Setter @Service @@ -65,6 +63,7 @@ public class TypeToEntityMigration5 extends Migration { .filter(t -> t.getType().equals("false_positive")) .findFirst(); if (falsePositive.isEmpty()) { + log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName()); return; } typeIdsToDelete.add(falsePositive.get().getId()); @@ -104,9 +103,7 @@ public class TypeToEntityMigration5 extends Migration { log.info("Finished processing type {}", typeEntity.getType()); }); - typeIdsToDelete.forEach(typeIdToDelete -> { - dictionaryController.deleteType(typeIdToDelete); - }); + typeIdsToDelete.forEach(typeIdToDelete -> dictionaryController.deleteType(typeIdToDelete)); log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId()); }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java index 2862dc876..faeb0a563 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java @@ -1,12 +1,5 @@ package com.iqser.red.service.peristence.v1.server.service; -import java.io.IOException; -import java.io.InputStream; - -import org.apache.commons.io.IOUtils; -import org.springframework.core.io.InputStreamResource; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; @@ -16,10 +9,15 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.SectionGrid; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.springframework.core.io.InputStreamResource; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.InputStream; @Slf4j @Service @@ -86,7 +84,7 @@ public class FileManagementStorageService { } - public ImportedRedactions getImportedRedactions(String dossierId, String fileId){ + public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { InputStreamResource inputStreamResource; try { inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS)); @@ -101,11 +99,11 @@ public class FileManagementStorageService { } } - public boolean typeExists(String dossierId, String fileId, FileType type) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, type)); + public boolean objectExists(String dossierId, String fileId, FileType origin) { + return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin)); } + public void deleteObject(String dossierId, String fileId, FileType fileType) { storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType)); @@ -117,4 +115,5 @@ public class FileManagementStorageService { storageService.deleteObject(storageId); } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 66993e0a0..2259f9299 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -32,7 +32,7 @@ public class FileStatusProcessingUpdateService { switch (analyzeResult.getMessageType()) { case STRUCTURE_ANALYSE: - if (settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) { + if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { fileStatusService.addToNerQueue(dossierId, fileId); } else { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index c9699d8d4..ff9d6a64d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,26 +1,10 @@ package com.iqser.red.service.peristence.v1.server.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; - -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import javax.transaction.Transactional; - -import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; -import com.iqser.red.service.peristence.v1.server.controller.RulesController; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; @@ -32,13 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Inter import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -46,9 +25,20 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @Slf4j @Service @@ -59,7 +49,6 @@ public class FileStatusService { private final DossierPersistenceService dossierPersistenceService; private final RabbitTemplate rabbitTemplate; private final ObjectMapper objectMapper; - private final RulesController rulesController; private final ManualRedactionProviderService manualRedactionProviderService; private final FileManagementStorageService fileManagementStorageService; private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; @@ -129,7 +118,6 @@ public class FileStatusService { } - public void setStatusProcessing(String fileId) { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING); @@ -199,9 +187,7 @@ public class FileStatusService { return; } - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS); - - if(requiresStructureAnalysis) { + if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); @@ -215,24 +201,28 @@ public class FileStatusService { protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var fileStatus = fileStatusPersistenceService.getStatus(fileId); + var fileEntity = fileStatusPersistenceService.getStatus(fileId); - if (fileStatus.isExcluded()) { - log.debug("File {} is excluded", fileStatus.getId()); + if (fileEntity.isExcluded()) { + log.debug("File {} is excluded", fileEntity.getId()); return; } - boolean reanalyse = isReanalyse(dossier, fileStatus); + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); - if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId,FileType.IMAGE_INFO)) { + + boolean reanalyse = fileModel.isReanalysisRequired(); + + if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); addToImageQueue(dossierId, fileId); return; } - MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); + MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus()); - if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) { + if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; @@ -241,26 +231,26 @@ public class FileStatusService { var analyseRequest = AnalyzeRequest.builder() .messageType(messageType) .dossierId(dossierId) - .analysisNumber(fileStatus.getNumberOfAnalyses() + 1) + .analysisNumber(fileModel.getNumberOfAnalyses() + 1) .sectionsToReanalyse(sectionsToReanalyse) .fileId(fileId) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) .dossierTemplateId(dossier.getDossierTemplateId()) - .lastProcessed(fileStatus.getLastProcessed()) - .fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId())) - .excludedPages(fileStatus.getExcludedPages()) + .lastProcessed(fileModel.getLastProcessed()) + .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) + .excludedPages(fileModel.getExcludedPages()) .build(); log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); - if(messageType.equals(MessageType.REANALYSE)) { + if (messageType.equals(MessageType.REANALYSE)) { setStatusProcessing(fileId); } else { setStatusFullProcessing(fileId); } try { - if(priority){ + if (priority) { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest)); } else { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest)); @@ -271,21 +261,10 @@ public class FileStatusService { } - private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) { - - return !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) - && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() - .isAfter(fileStatus.getLastFileAttributeChange())); - } - - private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus) { - if (!fileManagementStorageService.typeExists(dossierId, fileId, FileType.TEXT)) { + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { return MessageType.STRUCTURE_ANALYSE; } if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { @@ -416,7 +395,7 @@ public class FileStatusService { } - private void addToImageQueue(String dossierId, String fileId) { + public void addToImageQueue(String dossierId, String fileId) { setStatusImageAnalyzing(fileId); try { @@ -454,7 +433,6 @@ public class FileStatusService { } - public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) { fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index fd1aaa0e2..ff9716244 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -127,23 +127,26 @@ public class ManualRedactionService { response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build()); } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text reprocess(dossierId, fileId); + }else { + + var manualTextRedactions = convert(response.stream() + .map(r -> getAddRedaction(fileId, r.getAnnotationId())) + .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) + .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); + + if (!manualTextRedactions.isEmpty()) { + ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); + addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); + } + } - var manualTextRedactions = convert(response.stream() - .map(r -> getAddRedaction(fileId, r.getAnnotationId())) - .filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle()) - .collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper()); - - if (!manualTextRedactions.isEmpty()) { - ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build(); - addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - return response; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index 6fc58eb5d..1ebc00d9d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -33,16 +33,24 @@ public class ReanalysisRequiredStatusService { public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { - return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel)).iterator().next(); + return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), false).iterator().next(); + } + + public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel, boolean ignoreProcessingStates) { + return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), ignoreProcessingStates).iterator().next(); } public List enhanceFileStatusWithAnalysisRequirements(List fileModels) { + return enhanceFileStatusWithAnalysisRequirements(fileModels, false); + } + + public List enhanceFileStatusWithAnalysisRequirements(List fileModels, boolean ignoreProcessingStates) { Map> dossierTemplateVersionMap = new HashMap<>(); Map dossierVersionMap = new HashMap<>(); Map dossierMap = new HashMap<>(); fileModels.forEach(entry -> { - var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap); + var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap); entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired()); entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired()); }); @@ -52,15 +60,18 @@ public class ReanalysisRequiredStatusService { } private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus, + boolean ignoreProcessingStates, Map> dossierTemplateVersionMap, Map dossierVersionMap, Map dossierMap) { - if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) { return new AnalysisRequiredResult(false, true); } - if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) { + if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) + || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus()) + || ignoreProcessingStates) { switch (fileStatus.getWorkflowStatus()) { case NEW: diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java index 21ffc26ca..eeb92d83f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java @@ -9,11 +9,9 @@ import lombok.extern.slf4j.Slf4j; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.springframework.amqp.core.AmqpAdmin; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import javax.annotation.PostConstruct; import java.util.List; @Slf4j @@ -21,17 +19,9 @@ import java.util.List; @RequiredArgsConstructor public class AutomaticAnalysisJob implements Job { - @Value("${persistence-service.automaticAnalysis.pageFactor:500}") - private int pageFactor; - private final FileStatusService fileStatusService; private final AmqpAdmin amqpAdmin; private final FileManagementServiceSettings fileManagementServiceSettings; - - - @PostConstruct - protected void postConstruct() { - log.info("Automatic Analysis pageFactor: {}", pageFactor); - } + private final FileStatusService fileStatusService; @Override @@ -55,12 +45,11 @@ public class AutomaticAnalysisJob implements Job { var allStatusesIterator = allStatuses.iterator(); log.info("Files that require reanalysis: {}", allStatuses.size()); - var worstCaseScenarioQueuedPages = 0; + var queuedFiles = 0; - while (worstCaseScenarioQueuedPages < pageFactor * consumerCount && allStatusesIterator.hasNext()) { + while (queuedFiles < (consumerCount * 5)) { var next = allStatusesIterator.next(); // in case the file doesn't have numberOfPages set, we assume an average. - worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages(); if (next.isFullAnalysisRequired()) { log.info("Queued file: {} for automatic full analysis! ", next.getFilename()); @@ -69,6 +58,8 @@ public class AutomaticAnalysisJob implements Job { log.info("Queued file: {} for automatic reanalysis! ", next.getFilename()); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false); } + + queuedFiles++; } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 5c32f5f11..f2fb154a4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -1,38 +1,40 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; -import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.redaction.v1.model.ManualChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionType; -import feign.FeignException; -import org.assertj.core.util.Lists; -import org.junit.Test; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; - import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.redaction.v1.model.ManualChange; +import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - +import feign.FeignException; import lombok.SneakyThrows; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; + +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @@ -57,6 +59,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { @Autowired private FileManagementStorageService fileManagementStorageService; + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + @Autowired private ObjectMapper objectMapper; @@ -142,6 +147,49 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } + + @Test + @SneakyThrows + public void testAddToDictionaryRequiresReanalysis() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII"); + + + + // assume file is already proccessed once, test that add to dict triggers reanalysis + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8)); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8)); + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8)); + fileStatusPersistenceService.updateProcessingStatus(file.getId(),ProcessingStatus.PROCESSED); + + var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder() + .positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build())) + .section("section test") + .addToDictionary(true) + .addToDossierDictionary(false) + .status(AnnotationStatus.APPROVED) + .typeId(type.getId()) + .user("user") + .reason("1") + .value("test") + .legalBasis("1") + .rectangle(true) + .textAfter("Text After") + .textBefore("Text Before") + .sourceId("SourceId") + .build())).iterator().next(); + + + var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); + assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); + } + @Test @SneakyThrows public void testManualRedaction() { @@ -295,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .annotationStatus(AnnotationStatus.REQUESTED).build()); var loadedRemoveRedaction2 = manualRedactionClient.getRemoveRedaction(file.getId(), removeRedaction2.getAnnotationId()); assertThat(loadedRemoveRedaction2.getStatus()).isEqualTo(AnnotationStatus.REQUESTED); - assertThat(dictionaryClient.getDictionaryForType(type.getId(),null).getEntries().isEmpty()); + assertThat(dictionaryClient.getDictionaryForType(type.getId(), null).getEntries().isEmpty()); assertThat(loadedRemoveRedaction2.isRemoveFromDictionary()).isEqualTo(true); manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(), @@ -448,7 +496,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertThat(manualRedactions.getComments()).isNotEmpty(); assertThat(manualRedactions.getResizeRedactions()).isNotEmpty(); - List annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList()); + List annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList()); manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), annotationIds); // manualRedactions.getForceRedactions() // .forEach(e -> manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), List.of(e.getAnnotationId())));