From cf62a4f07b979c17812dec727576cee821cf7890 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 27 Jan 2022 10:51:54 +0200 Subject: [PATCH 1/4] RED-3242 - excluded from automatic analysis --- .../api/model/annotations/BaseAnnotation.java | 17 +++ .../v1/api/model/annotations/IdRemoval.java | 11 +- .../annotations/ManualForceRedaction.java | 11 +- .../ManualImageRecategorization.java | 11 +- .../annotations/ManualLegalBasisChange.java | 11 +- .../annotations/ManualRedactionEntry.java | 10 +- .../annotations/ManualResizeRedaction.java | 16 +-- .../v1/api/resources/ReanalysisResource.java | 18 +-- .../v1/api/resources/StatusResource.java | 5 + .../entity/annotations/IBaseAnnotation.java | 24 ++++ .../entity/annotations/IdRemovalEntity.java | 2 +- .../ManualForceRedactionEntity.java | 2 +- .../ManualImageRecategorizationEntity.java | 2 +- .../ManualLegalBasisChangeEntity.java | 2 +- .../ManualRedactionEntryEntity.java | 2 +- .../ManualResizeRedactionEntity.java | 21 +--- .../processor/entity/dossier/FileEntity.java | 3 + .../FileStatusPersistenceService.java | 44 ++++--- .../AddRedactionPersistenceService.java | 29 +++-- .../CommentPersistenceService.java | 2 +- .../ForceRedactionPersistenceService.java | 16 +-- ...ageRecategorizationPersistenceService.java | 12 +- .../LegalBasisChangePersistenceService.java | 14 ++- .../RemoveRedactionPersistenceService.java | 9 +- .../ResizeRedactionPersistenceService.java | 7 +- .../repository/FileRepository.java | 10 +- .../repository/ForceRedactionRepository.java | 7 +- .../ImageRecategorizationRepository.java | 8 +- .../LegalBasisChangeRepository.java | 2 - .../repository/ManualRedactionRepository.java | 9 +- .../repository/RemoveRedactionRepository.java | 3 + .../controller/FileStatusController.java | 7 ++ .../controller/ReanalysisController.java | 53 +++------ .../DictionarySearchAndNotifyService.java | 65 ----------- .../service/ExcludeFromAnalysisService.java | 16 ++- .../v1/server/service/FileService.java | 1 + .../FileStatusProcessingUpdateService.java | 3 +- .../v1/server/service/FileStatusService.java | 23 +++- .../ManualRedactionProviderService.java | 2 +- .../service/ManualRedactionService.java | 108 ++++++++++-------- .../scheduler/AutomaticAnalysisScheduler.java | 3 +- ...omatic-analysis-file-column.changelog.yaml | 24 ++++ .../integration/tests/ReanalysisTest.java | 15 +-- 43 files changed, 338 insertions(+), 322 deletions(-) create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/BaseAnnotation.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IBaseAnnotation.java rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/AddRedactionPersistenceService.java (82%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/CommentPersistenceService.java (98%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/ForceRedactionPersistenceService.java (84%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/ImageRecategorizationPersistenceService.java (84%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/LegalBasisChangePersistenceService.java (86%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/RemoveRedactionPersistenceService.java (89%) rename persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/{ => annotations}/ResizeRedactionPersistenceService.java (94%) delete mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionarySearchAndNotifyService.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/BaseAnnotation.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/BaseAnnotation.java new file mode 100644 index 000000000..3db93b763 --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/BaseAnnotation.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.service.v1.api.model.annotations; + +import lombok.Data; + +import java.time.OffsetDateTime; + +@Data +public class BaseAnnotation { + + private String annotationId; + private String fileId; + private String user; + private AnnotationStatus status; + private OffsetDateTime requestDate; + private OffsetDateTime processedDate; + private OffsetDateTime softDeletedTime; +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/IdRemoval.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/IdRemoval.java index dcf148f45..9b8f59e79 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/IdRemoval.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/IdRemoval.java @@ -5,21 +5,12 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.time.OffsetDateTime; - @Data @Builder @AllArgsConstructor @NoArgsConstructor -public class IdRemoval { +public class IdRemoval extends BaseAnnotation { - private String annotationId; - private String fileId; - private String user; - private AnnotationStatus status; private boolean removeFromDictionary; - private OffsetDateTime requestDate; - private OffsetDateTime processedDate; - private OffsetDateTime softDeletedTime; } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualForceRedaction.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualForceRedaction.java index 05474427b..5923182f3 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualForceRedaction.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualForceRedaction.java @@ -5,21 +5,12 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.time.OffsetDateTime; - @Data @Builder @AllArgsConstructor @NoArgsConstructor -public class ManualForceRedaction { +public class ManualForceRedaction extends BaseAnnotation { - private String annotationId; - private String fileId; - private String user; - private AnnotationStatus status; private String legalBasis; - private OffsetDateTime requestDate; - private OffsetDateTime processedDate; - private OffsetDateTime softDeletedTime; } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualImageRecategorization.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualImageRecategorization.java index b6726144f..de55bfb97 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualImageRecategorization.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualImageRecategorization.java @@ -5,20 +5,11 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.time.OffsetDateTime; - @Data @Builder @AllArgsConstructor @NoArgsConstructor -public class ManualImageRecategorization { +public class ManualImageRecategorization extends BaseAnnotation { - private String annotationId; - private String fileId; - private String user; - private AnnotationStatus status; private String type; - private OffsetDateTime requestDate; - private OffsetDateTime processedDate; - private OffsetDateTime softDeletedTime; } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualLegalBasisChange.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualLegalBasisChange.java index e5f352a41..67103201e 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualLegalBasisChange.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualLegalBasisChange.java @@ -5,23 +5,14 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.time.OffsetDateTime; - @Data @Builder @AllArgsConstructor @NoArgsConstructor -public class ManualLegalBasisChange { +public class ManualLegalBasisChange extends BaseAnnotation { - private String annotationId; - private String fileId; - private String user; - private AnnotationStatus status; private String section; private String value; private String legalBasis; - private OffsetDateTime requestDate; - private OffsetDateTime processedDate; - private OffsetDateTime softDeletedTime; } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualRedactionEntry.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualRedactionEntry.java index 346799614..320e62583 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualRedactionEntry.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualRedactionEntry.java @@ -5,7 +5,6 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -13,23 +12,16 @@ import java.util.List; @Builder @AllArgsConstructor @NoArgsConstructor -public class ManualRedactionEntry { +public class ManualRedactionEntry extends BaseAnnotation { - private String annotationId; - private String fileId; - private String user; private String type; private String value; private String reason; private String legalBasis; - private AnnotationStatus status; private String section; private boolean rectangle; private boolean addToDictionary; private boolean addToDossierDictionary; - private OffsetDateTime requestDate; - private OffsetDateTime processedDate; - private OffsetDateTime softDeletedTime; private List positions = new ArrayList<>(); private String textBefore; private String textAfter; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualResizeRedaction.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualResizeRedaction.java index 540a1ac1f..765a9bfdc 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualResizeRedaction.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ManualResizeRedaction.java @@ -1,27 +1,19 @@ package com.iqser.red.service.persistence.service.v1.api.model.annotations; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.List; + @Data @Builder @AllArgsConstructor @NoArgsConstructor -public class ManualResizeRedaction { +public class ManualResizeRedaction extends BaseAnnotation { - private String annotationId; - private String fileId; - private String user; - private AnnotationStatus status; - private OffsetDateTime requestDate; - private OffsetDateTime processedDate; - private OffsetDateTime softDeletedTime; private String value; @Builder.Default private List positions = new ArrayList<>(); diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java index 114546d8d..82230b33b 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java @@ -21,29 +21,31 @@ public interface ReanalysisResource { @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM) void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); - @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_PARAM) - void reanalyzeFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, - @RequestBody Set sectionsToReanalyse); - - @PostMapping(value = REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH) void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set fileIds, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM) - void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId); + void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser); @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_PARAM) - void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); + void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, + @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force); @PostMapping(value = OCR_REANALYZE_PATH + DOSSIER_ID_PATH_PARAM + BULK_REST_PATH) - void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set fileIds); + void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, + @RequestBody Set fileIds); @PostMapping(value = REINDEX_PATH) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java index 8bd2c646b..1efeebfac 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java @@ -24,6 +24,8 @@ public interface StatusResource { String EXCLUDED_STATUS_PARAM = "excluded"; + String EXCLUDED_FROM_AUTOMATIC_ANALYSIS_PARAM = "excludedFromAutomaticAnalysis"; + String APPROVER_ID_REQUEST_PARAM = "approverId"; String ASSIGNEE_ID_REQUEST_PARAM = "assigneeId"; String USER_ID_REQUEST_PARAM = "userId"; @@ -74,6 +76,9 @@ public interface StatusResource { @PostMapping(value = STATUS_PATH + "/toggle-analysis" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE) void toggleExclusion(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(EXCLUDED_STATUS_PARAM) boolean excluded); + @PostMapping(value = STATUS_PATH + "/toggle-automatic-analysis" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE) + void toggleAutomaticAnalysis(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(EXCLUDED_STATUS_PARAM) boolean excludedFromAutomaticAnalysis); + @PostMapping(value = STATUS_PATH + "/exclude-pages" + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE) void excludePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set pages); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IBaseAnnotation.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IBaseAnnotation.java new file mode 100644 index 000000000..303a193af --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IBaseAnnotation.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.annotations; + +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; + +import java.time.OffsetDateTime; + +public interface IBaseAnnotation { + + AnnotationEntityId getId(); + + AnnotationStatus getStatus(); + + OffsetDateTime getRequestDate(); + + void setRequestDate(OffsetDateTime requestedDate); + + OffsetDateTime getProcessedDate(); + + void setProcessedDate(OffsetDateTime processedDate); + + OffsetDateTime getSoftDeletedTime(); + + void setSoftDeletedTime(OffsetDateTime softDeletedTime); +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java index 19601aeb1..0663e223b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java @@ -16,7 +16,7 @@ import java.time.OffsetDateTime; @Builder @AllArgsConstructor @NoArgsConstructor -public class IdRemovalEntity { +public class IdRemovalEntity implements IBaseAnnotation { @EmbeddedId private AnnotationEntityId id; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java index a4f4fb7b2..5114b59a9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java @@ -16,7 +16,7 @@ import java.time.OffsetDateTime; @NoArgsConstructor @Entity @Table(name = "manual_force_redaction") -public class ManualForceRedactionEntity { +public class ManualForceRedactionEntity implements IBaseAnnotation { @EmbeddedId private AnnotationEntityId id; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java index 8e85d596f..d5a75faef 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java @@ -16,7 +16,7 @@ import java.time.OffsetDateTime; @NoArgsConstructor @Entity @Table(name = "manual_image_recategorization") -public class ManualImageRecategorizationEntity { +public class ManualImageRecategorizationEntity implements IBaseAnnotation { @EmbeddedId private AnnotationEntityId id; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java index f7aa9afc6..b5b319f10 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java @@ -16,7 +16,7 @@ import java.time.OffsetDateTime; @NoArgsConstructor @Entity @Table(name = "manual_legal_basis_change") -public class ManualLegalBasisChangeEntity { +public class ManualLegalBasisChangeEntity implements IBaseAnnotation { @EmbeddedId private AnnotationEntityId id; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java index 49bfa3ecf..31de806dc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java @@ -18,7 +18,7 @@ import java.util.List; @AllArgsConstructor @NoArgsConstructor @Table(name = "manual_redaction") -public class ManualRedactionEntryEntity { +public class ManualRedactionEntryEntity implements IBaseAnnotation { @EmbeddedId private AnnotationEntityId id; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java index ff54453ee..6bb03cc35 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java @@ -1,33 +1,24 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotations; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; - -import javax.persistence.Column; -import javax.persistence.ElementCollection; -import javax.persistence.EmbeddedId; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.ManyToOne; -import javax.persistence.Table; - import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import javax.persistence.*; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + @Data @Builder @AllArgsConstructor @NoArgsConstructor @Entity @Table(name = "manual_resize_redaction") -public class ManualResizeRedactionEntity { +public class ManualResizeRedactionEntity implements IBaseAnnotation { @EmbeddedId private AnnotationEntityId id; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index ea25aaed0..5cd4bcabf 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -118,6 +118,9 @@ public class FileEntity { @Column private boolean excluded; + @Column + private boolean excludedFromAutomaticAnalysis; + @Column private OffsetDateTime hardDeletedTime; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 27e05ef03..351c350a7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -71,7 +71,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateFlags(fileId, OffsetDateTime.now(), hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); + fileRepository.updateFlags(fileId, OffsetDateTime.now(), hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); } @@ -81,9 +81,8 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateWorkflowStatus(fileId, workflowStatus, - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null); + fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), + approval ? OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS) : null, approval); } @@ -97,9 +96,9 @@ public class FileStatusPersistenceService { if (processingStatus == ProcessingStatus.PROCESSED) { // In case the file is updated to "processed", "lastProcessed" date should be updated to "now" fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } else { - fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } } @@ -112,7 +111,7 @@ public class FileStatusPersistenceService { } fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } @@ -122,7 +121,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateLastOCRTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time); + fileRepository.updateLastOCRTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), time); } @@ -132,7 +131,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateHasComments(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasComments); + fileRepository.updateHasComments(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasComments); } @@ -142,7 +141,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateLastManualRedaction(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date); + fileRepository.updateLastManualRedaction(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date); } @@ -154,7 +153,7 @@ public class FileStatusPersistenceService { return; } fileRepository.setUpdateLastManualRedactionAndHasSuggestions(fileId, - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), date, hasSuggestions); } @@ -171,7 +170,7 @@ public class FileStatusPersistenceService { var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes); file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - file.setFileAttributes(fileAttributeEntities); + file.setFileAttributes(fileAttributeEntities); }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); @@ -188,7 +187,7 @@ public class FileStatusPersistenceService { file.setLastManualRedaction(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - file.setExcludedPages(excludedPages); + file.setExcludedPages(excludedPages); }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); @@ -224,7 +223,7 @@ public class FileStatusPersistenceService { public void softDelete(String fileId, OffsetDateTime softDeletedTime) { int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.DELETED, - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -237,7 +236,7 @@ public class FileStatusPersistenceService { int countUpdate = fileRepository.setHardDelete(fileId, ProcessingStatus.DELETED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -279,7 +278,18 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + if (countUpdate == 0) { + throw new NotFoundException("Unknown file=" + fileId); + } + } + + @Transactional + public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) { + if (isFileDeleted(fileId)) { + return; + } + int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -296,7 +306,7 @@ public class FileStatusPersistenceService { int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java similarity index 82% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/AddRedactionPersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index f94de45c1..454bf3cff 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; @@ -16,9 +16,9 @@ import javax.transaction.Transactional; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; @Service @RequiredArgsConstructor @@ -28,7 +28,7 @@ public class AddRedactionPersistenceService { private final ManualRedactionRepository manualRedactionRepository; - public void insert(String fileId, String annotationId, AddRedactionRequest addRedactionRequest) { + public ManualRedactionEntryEntity insert(String fileId, String annotationId, AddRedactionRequest addRedactionRequest) { ManualRedactionEntryEntity manualRedactionEntry = new ManualRedactionEntryEntity(); manualRedactionEntry.setId(new AnnotationEntityId(annotationId, fileId)); @@ -37,13 +37,18 @@ public class AddRedactionPersistenceService { manualRedactionEntry.setPositions(convert(addRedactionRequest.getPositions())); manualRedactionEntry.setTypeId(addRedactionRequest.getTypeId()); - manualRedactionRepository.save(manualRedactionEntry); + if (addRedactionRequest.getStatus() == AnnotationStatus.APPROVED) { + manualRedactionEntry.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } + + + return manualRedactionRepository.save(manualRedactionEntry); } @Transactional - public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter){ + public void updateSurroundingText(AnnotationEntityId id, String textBefore, String textAfter) { manualRedactionRepository.updateSurroundingText(id, textBefore, textAfter); } @@ -69,7 +74,7 @@ public class AddRedactionPersistenceService { public Set findAddRedactions(String fileId, boolean includeDeletions) { - return manualRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions).stream().collect(Collectors.toSet()); + return new HashSet<>(manualRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions)); } @@ -94,19 +99,13 @@ public class AddRedactionPersistenceService { @Transactional public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) { - manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - annotationStatus); + manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus); } @Transactional - public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary - , boolean isAddOrRemoveFromDossierDictionary) { + public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus, boolean isAddOrRemoveFromDictionary, boolean isAddOrRemoveFromDossierDictionary) { - manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - annotationStatus, - isAddOrRemoveFromDictionary, isAddOrRemoveFromDossierDictionary); + manualRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, isAddOrRemoveFromDictionary, isAddOrRemoveFromDossierDictionary); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/CommentPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java similarity index 98% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/CommentPersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java index 69daa7511..3495bb625 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/CommentPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ForceRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java similarity index 84% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ForceRedactionPersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java index 013f4137a..db1d1c4d7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ForceRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java @@ -1,12 +1,14 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; import javax.transaction.Transactional; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -33,17 +35,12 @@ public class ForceRedactionPersistenceService { manualForceRedaction.setId(new AnnotationEntityId(forceRedactionRequest.getAnnotationId(), fileId)); BeanUtils.copyProperties(forceRedactionRequest, manualForceRedaction); manualForceRedaction.setRequestDate(OffsetDateTime.now()); - if(manualForceRedaction.getStatus().equals(AnnotationStatus.APPROVED)) { - manualForceRedaction.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - } - forceRedactionRepository.save(manualForceRedaction); } @Transactional public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) { - forceRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, - OffsetDateTime.now()); + forceRedactionRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus); } @@ -72,7 +69,10 @@ public class ForceRedactionPersistenceService { public Set findForceRedactions(String fileId, boolean includeDeletions) { - return forceRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions).stream().collect(Collectors.toSet()); + return new HashSet<>(forceRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions)); } + public void markAsProcessed(ManualForceRedaction e) { + forceRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ImageRecategorizationPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java similarity index 84% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ImageRecategorizationPersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java index ac6a2ec9b..1e7e2257f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ImageRecategorizationPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ImageRecategorizationPersistenceService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; @@ -6,12 +6,15 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ImageRecategorizationRepository; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization; import lombok.RequiredArgsConstructor; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import javax.transaction.Transactional; import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.List; @Service @@ -34,8 +37,7 @@ public class ImageRecategorizationPersistenceService { @Transactional public void updateStatus(String fileId, String annotationId, AnnotationStatus annotationStatus) { - imageRecategorizationRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus, - OffsetDateTime.now()); + imageRecategorizationRepository.updateStatus(new AnnotationEntityId(annotationId, fileId), annotationStatus); } @Transactional @@ -68,5 +70,7 @@ public class ImageRecategorizationPersistenceService { } - + public void markAsProcessed(ManualImageRecategorization e) { + imageRecategorizationRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/LegalBasisChangePersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java similarity index 86% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/LegalBasisChangePersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java index f809e97ee..0dec0c75a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/LegalBasisChangePersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import javax.transaction.Transactional; import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.Set; import java.util.stream.Collectors; @@ -22,13 +23,18 @@ public class LegalBasisChangePersistenceService { private final LegalBasisChangeRepository legalBasisChangeRepository; - public void insert(String fileId, LegalBasisChangeRequest legalBasisChangeRequest) { + public ManualLegalBasisChangeEntity insert(String fileId, LegalBasisChangeRequest legalBasisChangeRequest) { ManualLegalBasisChangeEntity manualLegalBasisChange = new ManualLegalBasisChangeEntity(); manualLegalBasisChange.setId(new AnnotationEntityId(legalBasisChangeRequest.getAnnotationId(), fileId)); BeanUtils.copyProperties(legalBasisChangeRequest, manualLegalBasisChange); - manualLegalBasisChange.setRequestDate(OffsetDateTime.now()); - legalBasisChangeRepository.save(manualLegalBasisChange); + manualLegalBasisChange.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + + if(legalBasisChangeRequest.getStatus() == AnnotationStatus.APPROVED) { + manualLegalBasisChange.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } + + return legalBasisChangeRepository.save(manualLegalBasisChange); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/RemoveRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java similarity index 89% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/RemoveRedactionPersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java index 5c1476b09..b24d7bf86 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/RemoveRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java @@ -1,10 +1,11 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; import lombok.RequiredArgsConstructor; import org.springframework.beans.BeanUtils; @@ -12,6 +13,7 @@ import org.springframework.stereotype.Service; import javax.transaction.Transactional; import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.Set; import java.util.stream.Collectors; @@ -74,5 +76,8 @@ public class RemoveRedactionPersistenceService { annotationStatus, OffsetDateTime.now(), isAddOrRemoveFromDictionary); } - + @Transactional + public void markAsProcessed(IdRemoval e) { + removeRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ResizeRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java similarity index 94% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ResizeRedactionPersistenceService.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java index d1a8ca32a..95e692524 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ResizeRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.persistence; +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations; import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @@ -29,18 +29,19 @@ public class ResizeRedactionPersistenceService { @Transactional - public void insert(String fileId, ResizeRedactionRequest resizeRedactionRequest) { + public ManualResizeRedactionEntity insert(String fileId, ResizeRedactionRequest resizeRedactionRequest) { ManualResizeRedactionEntity manualResizeRedaction = new ManualResizeRedactionEntity(); manualResizeRedaction.setId(new AnnotationEntityId(resizeRedactionRequest.getAnnotationId(), fileId)); BeanUtils.copyProperties(resizeRedactionRequest, manualResizeRedaction); manualResizeRedaction.setPositions(convert(resizeRedactionRequest.getPositions(), RectangleEntity.class)); manualResizeRedaction.setRequestDate(OffsetDateTime.now()); + if(manualResizeRedaction.getStatus().equals(AnnotationStatus.APPROVED)) { manualResizeRedaction.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } - resizeRedactionRepository.save(manualResizeRedaction); + return resizeRedactionRepository.save(manualResizeRedaction); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 7ce3ab05d..94c5099f2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -45,9 +45,9 @@ public interface FileRepository extends JpaRepository { int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber); @Modifying - @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + - "where f.id = :fileId") - void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate); + @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") + void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate, boolean excludedFromAutomaticAnalysis); @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " + @@ -109,6 +109,10 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.excluded = :excluded, f.lastUpdated = :lastUpdated where f.id = :fileId") int toggleExclusion(String fileId, boolean excluded, OffsetDateTime lastUpdated); + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId") + int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated); + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " + diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ForceRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ForceRedactionRepository.java index 47d2c63c5..3166f913f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ForceRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ForceRedactionRepository.java @@ -16,9 +16,9 @@ public interface ForceRedactionRepository extends JpaRepository findByIdFileId(String fileId); @Modifying - @Query("update ManualForceRedactionEntity mfr set mfr.status = :annotationStatus, mfr.processedDate = :processedDate " + + @Query("update ManualForceRedactionEntity mfr set mfr.status = :annotationStatus " + "where mfr.id = :annotationEntityId") - void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus, OffsetDateTime processedDate); + void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus); @Modifying @Query("update ManualForceRedactionEntity mfr set mfr.softDeletedTime = :softDeletedTime " + @@ -31,4 +31,7 @@ public interface ForceRedactionRepository extends JpaRepository findByFileIdIncludeDeletions(String fileId, boolean includeDeletions); + @Modifying + @Query("update ManualForceRedactionEntity mfr set mfr.processedDate = :processedDate where mfr.id = :annotationEntityId") + void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ImageRecategorizationRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ImageRecategorizationRepository.java index 2ef2eadb4..eb7dc30b2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ImageRecategorizationRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ImageRecategorizationRepository.java @@ -15,9 +15,9 @@ public interface ImageRecategorizationRepository extends JpaRepository findByIdFileId(String fileId); @Modifying - @Query("update ManualImageRecategorizationEntity mir set mir.status = :annotationStatus, mir.processedDate = :processedDate " + + @Query("update ManualImageRecategorizationEntity mir set mir.status = :annotationStatus " + "where mir.id = :annotationEntityId") - void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus, OffsetDateTime processedDate); + void updateStatus(AnnotationEntityId annotationEntityId, AnnotationStatus annotationStatus); @Modifying @Query("update ManualImageRecategorizationEntity mir set mir.softDeletedTime = :softDeletedTime " + @@ -29,4 +29,8 @@ public interface ImageRecategorizationRepository extends JpaRepository findByFileIdIncludeDeletions(String fileId, boolean includeDeletions); + + @Modifying + @Query("update ManualImageRecategorizationEntity mir set mir.processedDate = :processedDate where mir.id = :annotationEntityId") + void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/LegalBasisChangeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/LegalBasisChangeRepository.java index 146115755..094086ecc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/LegalBasisChangeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/LegalBasisChangeRepository.java @@ -13,8 +13,6 @@ import java.util.Optional; public interface LegalBasisChangeRepository extends JpaRepository { - List findByIdFileId(String fileId); - @Modifying @Query("update ManualLegalBasisChangeEntity mlbc set mlbc.status = :annotationStatus, mlbc.processedDate = :processedDate " + "where mlbc.id = :annotationEntityId") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ManualRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ManualRedactionRepository.java index dd1e41068..10c793f59 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ManualRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ManualRedactionRepository.java @@ -20,14 +20,13 @@ public interface ManualRedactionRepository extends JpaRepository findByFileIdIncludeDeletions(String fileId, boolean includeDeletions); + @Modifying + @Query("update IdRemovalEntity idr set idr.processedDate = :processedDate where idr.id = :annotationEntityId") + void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java index 8487f671f..56584703c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java @@ -149,6 +149,13 @@ public class FileStatusController implements StatusResource { } + public void toggleAutomaticAnalysis(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @PathVariable(FILE_ID) String fileId, @RequestParam(EXCLUDED_STATUS_PARAM) boolean excludedFromAutomaticAnalysis) { + + excludeFromAnalysis.toggleAutomaticAnalysis(dossierId, fileId, excludedFromAutomaticAnalysis); + + } + public void excludePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set pages) { FileEntity fileStatus = fileStatusService.getStatus(fileId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index 44f838871..291a7fb86 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -41,51 +41,35 @@ public class ReanalysisController implements ReanalysisResource { @Override public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { - var relevantFiles = getRelevantFiles(dossierId); - + var relevantFiles = getRelevantFiles(dossierId, triggeredByUser); reanalyseFiles(dossierId, force, relevantFiles); } - @Override - public void reanalyzeFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, - @PathVariable(FILE_ID_PARAM) String fileId, - @RequestBody Set sectionsToReanalyse) { - - log.info("Reanalyse: {} / {} with sections: {}", dossierId, fileId, sectionsToReanalyse); - if (sectionsToReanalyse != null) { - var relevantFiles = getRelevantFiles(dossierId, Sets.newHashSet(fileId)); - relevantFiles.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, fileId, 10, sectionsToReanalyse); - }); - } else { - this.reanalyzeFiles(dossierId, Sets.newHashSet(fileId), false); - } - } - - @Override public void reanalyzeFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set fileIds, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { - var relevantFiles = getRelevantFiles(dossierId, fileIds); + var relevantFiles = getRelevantFiles(dossierId, fileIds, triggeredByUser); reanalyseFiles(dossierId, force, relevantFiles); } - public void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId) { + public void ocrDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser) { - var relevantFiles = getRelevantFiles(dossierId); + var relevantFiles = getRelevantFiles(dossierId, triggeredByUser); relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId())); } - public void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set fileIds) { + public void ocrFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, @RequestBody Set fileIds) { - var relevantFiles = getRelevantFiles(dossierId, fileIds); + var relevantFiles = getRelevantFiles(dossierId, fileIds, triggeredByUser); relevantFiles.stream().filter(fileStatus -> fileStatus.getLastOCRTime() == null).forEach(fileStatus -> fileStatusService.setStatusOcrProcessing(dossierId, fileStatus.getId())); } @@ -96,7 +80,9 @@ public class ReanalysisController implements ReanalysisResource { } - public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { + public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId, + @RequestParam(value = "triggeredByUser", required = false, defaultValue = FALSE) boolean triggeredByUser, + @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { dossierPersistenceService.getAndValidateDossier(dossierId); @@ -110,26 +96,27 @@ public class ReanalysisController implements ReanalysisResource { if (dossierFile.getLastOCRTime() != null) { throw new ConflictException("File already has been OCR processed"); } - ocrFiles(dossierId, Sets.newHashSet(fileId)); + ocrFiles(dossierId,triggeredByUser, Sets.newHashSet(fileId)); } } - private List getRelevantFiles(String dossierId) { + private List getRelevantFiles(String dossierId, boolean triggeredByUser) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); return fileStatusService.getDossierStatus(dossier.getId()) .stream() .filter(fileStatus -> !fileStatus.getProcessingStatus().equals(ProcessingStatus.DELETED)) .filter(fileStatus -> !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) + .filter(fileStatus -> triggeredByUser || !fileStatus.isExcludedFromAutomaticAnalysis()) .collect(Collectors.toList()); } - private List getRelevantFiles(String dossierId, Collection fileIds) { + private List getRelevantFiles(String dossierId, Collection fileIds, boolean triggeredByUser) { - var relevantDossierFiles = getRelevantFiles(dossierId); + var relevantDossierFiles = getRelevantFiles(dossierId, triggeredByUser); var relevantFiles = new ArrayList(); for (var fileId : fileIds) { @@ -140,10 +127,6 @@ public class ReanalysisController implements ReanalysisResource { var dossierFile = dossierFileOptional.get(); - if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { - throw new ConflictException("Cannot analyse a deleted/approved file"); - } - relevantFiles.add(dossierFile); } @@ -155,14 +138,14 @@ public class ReanalysisController implements ReanalysisResource { if (force) { filesToReanalyse.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), 2); + fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true); }); } else { var enhancedAndConvertedFiles = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert( fileStatusService.getAllStatuses(), FileModel.class, new FileModelMapper())); enhancedAndConvertedFiles.forEach(file -> { - fileStatusService.setStatusReprocess(dossierId, file.getId(), 2); + fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true); }); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionarySearchAndNotifyService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionarySearchAndNotifyService.java deleted file mode 100644 index f47baeb67..000000000 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionarySearchAndNotifyService.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.iqser.red.service.peristence.v1.server.service; - -import com.iqser.red.service.peristence.v1.server.client.SearchClient; -import com.iqser.red.service.peristence.v1.server.controller.ReanalysisController; -import com.iqser.red.service.search.v1.model.MatchedSection; -import com.iqser.red.service.search.v1.model.SearchRequest; -import lombok.Builder; -import lombok.Data; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -@Service -@RequiredArgsConstructor -public class DictionarySearchAndNotifyService { - - private final ReanalysisController reanalysisClient; - private final SearchClient searchClient; - - public void dictionaryEntriesUpdated(String dossierTemplateId, Collection changedEntries) { - - log.info("Detect Changes for dossierTemplateId: {} and entries: {}", dossierTemplateId, changedEntries); - - Map> searchResultMap = new HashMap<>(); - - for (var entry : changedEntries) { - - - var searchResult = searchClient.getDossierStatus(SearchRequest.builder() - .dossierTemplateIds(Collections.singletonList(dossierTemplateId)).page(1).pageSize(10_000) - .returnSections(true) - .queryString(entry) - .build()); - - log.info("Got changes: {} for entry: {}", searchResult.getTotal(), entry); - - searchResult.getMatchedDocuments().forEach(document -> { - - var currentSections = searchResultMap.computeIfAbsent(SearchKey.builder().fileId(document.getFileId()).dossierId(document.getDossierId()).build(), (data) -> new HashSet<>()); - currentSections.addAll(document.getMatchedSections().stream().map(MatchedSection::getSectionNumber).collect(Collectors.toList())); - - }); - } - - - log.info("Should reanalyse: {}", searchResultMap); - searchResultMap.forEach((searchKey, sectionNumbers) -> { - log.info("Reanalyse: {} / {} ", searchKey, sectionNumbers); - reanalysisClient.reanalyzeFile(searchKey.getDossierId(), searchKey.getFileId(), sectionNumbers); - }); - - } - - @Data - @Builder - public static class SearchKey { - private String fileId; - private String dossierId; - } - -} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java index 49f7391d5..ad252f754 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java @@ -9,10 +9,8 @@ import org.springframework.stereotype.Service; @RequiredArgsConstructor public class ExcludeFromAnalysisService { - private final FileStatusService fileStatusService; - public void toggleExclusion(String dossierId, String fileId, boolean excluded) { var status = fileStatusService.getStatus(fileId); @@ -31,4 +29,18 @@ public class ExcludeFromAnalysisService { } + + public void toggleAutomaticAnalysis(String dossierId, String fileId, boolean excludedFromAutomaticAnalysis) { + + var status = fileStatusService.getStatus(fileId); + + // toggle status + fileStatusService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis); + + if(!excludedFromAutomaticAnalysis){ + // if file has been re-enabled - process it + fileStatusService.setStatusFullReprocess(dossierId, fileId, 2); + } + + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 925538735..d7e3fdbd5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 7ec7e0080..b2a07faa1 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -31,7 +31,7 @@ public class FileStatusProcessingUpdateService { case SURROUNDING_TEXT: fileStatusService.setStatusProcessed(analyzeResult.getFileId()); - manualRedactionService.updateManualRedactions(fileId, analyzeResult.getManualRedactions()); + manualRedactionService.updateSurroundingText(fileId, analyzeResult.getManualRedactions()); break; case REANALYSE: case FULL_ANALYSE: @@ -47,6 +47,7 @@ public class FileStatusProcessingUpdateService { indexingService.addToIndexingQueue(IndexMessageType.INSERT, dossier.getDossierTemplateId(), dossierId, fileId, 2); } + manualRedactionService.updateProcessedDate(fileId, analyzeResult.getManualRedactions()); analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 5cb8129de..51cf70916 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -12,13 +12,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; -import com.iqser.red.service.search.v1.model.IndexMessageType; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -128,12 +128,18 @@ public class FileStatusService { @Transactional public void setStatusReprocess(String dossierId, String fileId, int priority) { - setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet()); + setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), false); + } + + @Transactional + public void setStatusReprocess(String dossierId, String fileId, int priority, boolean triggeredManually) { + + setStatusReprocess(dossierId, fileId, priority, Sets.newHashSet(), triggeredManually); } @Transactional - public void setStatusReprocess(String dossierId, String fileId, int priority, Set sectionsToReanalyse) { + public void setStatusReprocess(String dossierId, String fileId, int priority, Set sectionsToReanalyse, boolean triggeredManually) { log.info("Reprocessing file: {} from dossier {}", fileId, dossierId); @@ -144,6 +150,11 @@ public class FileStatusService { return; } + if(fileStatus.isExcludedFromAutomaticAnalysis() && !triggeredManually){ + log.debug("File {} is excluded from automatic analysis", fileStatus.getId()); + return; + } + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS); addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse); } @@ -241,6 +252,12 @@ public class FileStatusService { } + public void toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis) { + + fileStatusPersistenceService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis); + } + + @Transactional protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set sectionsToReanalyse) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionProviderService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionProviderService.java index 3b61b4d50..87da76d95 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionProviderService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionProviderService.java @@ -5,7 +5,7 @@ import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper; import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 2b1a876ba..b3327e968 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -13,21 +13,22 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.MessageType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; - import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; +import javax.transaction.Transactional; import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -64,6 +65,7 @@ public class ManualRedactionService { private final HashFunction hashFunction = Hashing.murmur3_128(); + @Transactional public ManualAddResponse addAddRedaction(String dossierId, String fileId, AddRedactionRequest addRedactionRequest) { dossierPersistenceService.getAndValidateDossier(dossierId); @@ -83,6 +85,7 @@ public class ManualRedactionService { String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString(); OffsetDateTime now = OffsetDateTime.now(); + addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest); Long commentId = null; @@ -110,7 +113,6 @@ public class ManualRedactionService { return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build(); } - public ManualAddResponse addRemoveRedaction(String dossierId, String fileId, RemoveRedactionRequest removeRedactionRequest) { @@ -138,7 +140,7 @@ public class ManualRedactionService { } if (removeRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED)) { - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } return ManualAddResponse.builder() @@ -171,7 +173,7 @@ public class ManualRedactionService { analysisFlagsCalculationService.calculateFlags(dossierId, fileId); if (forceRedactionRequest.getStatus().equals(AnnotationStatus.APPROVED)) { - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } return ManualAddResponse.builder() @@ -228,7 +230,7 @@ public class ManualRedactionService { fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, now, true); } else { fileStatusPersistenceService.updateLastManualRedaction(fileId, now); - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); @@ -240,17 +242,6 @@ public class ManualRedactionService { } - public CommentEntity addComment(String fileId, String annotationId, CommentRequest commentRequest) { - - var createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser()); - - fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); - fileStatusPersistenceService.updateHasComments(fileId, true); - - return createdComment; - } - - public ManualAddResponse addResizeRedaction(String dossierId, String fileId, ResizeRedactionRequest resizeRedactionRequest) { @@ -283,6 +274,16 @@ public class ManualRedactionService { .build(); } + public CommentEntity addComment(String fileId, String annotationId, CommentRequest commentRequest) { + + var createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser()); + + fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); + fileStatusPersistenceService.updateHasComments(fileId, true); + + return createdComment; + } + public ManualRedactionEntryEntity getAddRedaction(String fileId, String annotationId) { @@ -408,7 +409,7 @@ public class ManualRedactionService { fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions); } else { fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -456,7 +457,7 @@ public class ManualRedactionService { .filter(entry -> entry.getId().equals(idRemoval.getId().getAnnotationId())) .findFirst(); - if (!redactionLogEntryOptional.isPresent()) { + if (redactionLogEntryOptional.isEmpty()) { throw new NotFoundException("Annotation does not exist in redaction log."); } @@ -482,16 +483,6 @@ public class ManualRedactionService { } - private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) { - - if (redactionLogEntry.isDossierDictionaryEntry()) { - return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()); - } else { - return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId()); - } - } - - public void updateForceRedactionStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) { @@ -503,6 +494,7 @@ public class ManualRedactionService { } + @Transactional public void updateLegalBasisChangeStatus(String dossierId, String fileId, String annotationId, AnnotationStatus annotationStatus) { @@ -523,10 +515,10 @@ public class ManualRedactionService { // if it was previously approved, revert the delete if (imageRecategorization.getStatus() == AnnotationStatus.APPROVED) { - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } } else if (annotationStatus.equals(AnnotationStatus.APPROVED)) { - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } dossierPersistenceService.getAndValidateDossier(dossierId); @@ -581,6 +573,40 @@ public class ManualRedactionService { } + @Transactional + public void updateSurroundingText(String fileId, ManualRedactions manualRedactions) { + + // These are marked as processed once surrounding text is computed ( TBD if this is correct ? ) + manualRedactions.getEntriesToAdd().forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter())); + manualRedactions.getResizeRedactions().forEach(e -> resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter())); + } + + + @Transactional + public void updateProcessedDate(String fileId, ManualRedactions manualRedactions) { + + // These are marked as processed once analysis completes, not when they are set as approved + manualRedactions.getIdsToRemove().forEach(removeRedactionPersistenceService::markAsProcessed); + manualRedactions.getForceRedactions().forEach(forceRedactionPersistenceService::markAsProcessed); + manualRedactions.getImageRecategorization().forEach(recategorizationPersistenceService::markAsProcessed); + + } + + + private void reprocess(String dossierId, String fileId) { + fileStatusService.setStatusReprocess(dossierId, fileId, 2); + } + + + private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) { + + if (redactionLogEntry.isDossierDictionaryEntry()) { + return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId(), dossier.getId()); + } else { + return toTypeId(redactionLogEntry.getType(), dossier.getDossierTemplateId()); + } + } + private void handleAddToDictionary(String fileId, String annotationId, String typeId, String value, AnnotationStatus status, boolean addToDictionary, boolean addToDossierDictionary, boolean revert, String dossierId) { @@ -615,7 +641,7 @@ public class ManualRedactionService { .filter(entry -> entry.getId().equals(annotationId)) .findFirst(); - if (!redactionLogEntryOptional.isPresent()) { + if (redactionLogEntryOptional.isEmpty()) { throw new NotFoundException("Annotation does not exist in redaction log."); } @@ -638,7 +664,7 @@ public class ManualRedactionService { log.debug("Adding entries to {} for {} / {}", typeId, dossierId, fileId); dictionaryController.addEntries(typeId, List.of(value), false, false); - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } catch (Exception e) { throw new BadRequestException(e.getMessage()); } @@ -650,7 +676,7 @@ public class ManualRedactionService { try { log.debug("Deleting entries to {} for {} / {}", typeId, dossierId, fileId); dictionaryController.deleteEntries(typeId, List.of(value)); - fileStatusService.setStatusReprocess(dossierId, fileId, 2); + reprocess(dossierId, fileId); } catch (FeignException e) { throw new BadRequestException(e.getMessage()); } @@ -711,18 +737,6 @@ public class ManualRedactionService { } - public void updateManualRedactions(String fileId, ManualRedactions manualRedactions) { - - manualRedactions.getEntriesToAdd().forEach(e -> { - addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()); - }); - - manualRedactions.getResizeRedactions().forEach(e -> { - resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()); - }); - } - - private void addManualRedactionToAnalysisQueue(String dossierId, String fileId, ManualRedactions manualRedactions) { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java index 6ff77d069..a243b5f22 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java @@ -67,7 +67,7 @@ public class AutomaticAnalysisScheduler { if (next.isFullAnalysisRequired()) { log.info("Queued file: {} for automatic full analysis! ", next.getFilename()); fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), 1); - } else if(next.isReanalysisRequired()){ + } else if (next.isReanalysisRequired()) { log.info("Queued file: {} for automatic reanalysis! ", next.getFilename()); fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), 1); } @@ -86,6 +86,7 @@ public class AutomaticAnalysisScheduler { .stream() .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) .filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) + .filter(f -> !f.isExcludedFromAutomaticAnalysis()) .filter(f -> !isProcessing(f)) .filter(f -> f.getDossier().getSoftDeletedTime() == null && f.getDossier().getHardDeletedTime() == null && f.getDossier().getArchivedTime() == null) .collect(Collectors.toList()), FileModel.class, new FileModelMapper())) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml new file mode 100644 index 000000000..36ed9537d --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml @@ -0,0 +1,24 @@ +databaseChangeLog: + - changeSet: + id: add-excluded-from-automatic-analysis-column + author: timo + changes: + - addColumn: + columns: + - column: + name: excluded_from_automatic_analysis + type: BOOLEAN + defaultValue: false + tableName: file + - changeSet: + id: updated-excluded-from-automatic-analysis-column + author: timo + changes: + - update: + columns: + - column: + name: excluded_from_automatic_analysis + value: true + schemaName: public + tableName: file + where: workflow_status = 'APPROVED' diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java index 615b95c59..6d891e324 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java @@ -42,33 +42,28 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); - reanalysisClient.ocrDossier(dossier.getId()); + reanalysisClient.ocrDossier(dossier.getId(),true); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); resetProcessingStatus(file); - reanalysisClient.ocrFile(dossier.getId(), file.getId(), true); + reanalysisClient.ocrFile(dossier.getId(), file.getId(),true, true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); resetProcessingStatus(file); - reanalysisClient.ocrFiles(dossier.getId(), Set.of(file.getId())); + reanalysisClient.ocrFiles(dossier.getId(),true, Set.of(file.getId())); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING); resetProcessingStatus(file); - reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true); + reanalysisClient.reanalyzeFiles(dossier.getId(), Set.of(file.getId()), true,true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); resetProcessingStatus(file); - reanalysisClient.reanalyzeFile(dossier.getId(), file.getId(), Sets.newHashSet()); - loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); - assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); - resetProcessingStatus(file); - - reanalysisClient.reanalyzeDossier(dossier.getId(), true); + reanalysisClient.reanalyzeDossier(dossier.getId(), true,true); loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING); resetProcessingStatus(file); From d14ee9984c8d7f8a2b8d73c92bfdeea1ca9f5846 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 27 Jan 2022 11:14:11 +0200 Subject: [PATCH 2/4] Final touches for RED-3242 --- ...automatic-analysis-file-column.changelog.yaml} | 0 .../db/changelog/db.changelog-master.yaml | 2 ++ .../integration/tests/FileProcessingTest.java | 15 +++++++++------ .../integration/tests/ManualRedactionTest.java | 3 +-- 4 files changed, 12 insertions(+), 8 deletions(-) rename persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/{5-excluded-from-automatic-analysis-file-column.changelog.yaml => 6-excluded-from-automatic-analysis-file-column.changelog.yaml} (100%) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/6-excluded-from-automatic-analysis-file-column.changelog.yaml similarity index 100% rename from persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml rename to persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/6-excluded-from-automatic-analysis-file-column.changelog.yaml diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 889dce98c..872f2026f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -9,3 +9,5 @@ databaseChangeLog: file: db/changelog/4-archived-dossier.changelog.yaml - include: file: db/changelog/5-imported-annotation.changelog.yaml + - include: + file: db/changelog/6-excluded-from-automatic-analysis-file-column.changelog.yaml diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java index 6541f3234..dab2cb437 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileProcessingTest.java @@ -1,20 +1,20 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; - -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileProcessingClient; import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import static org.assertj.core.api.Assertions.assertThat; public class FileProcessingTest extends AbstractPersistenceServerServiceTest { @@ -48,6 +48,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder() + .manualRedactions(new ManualRedactions()) .messageType(MessageType.FULL_ANALYSE) .analysisVersion(100) .fileId(file.getId()) @@ -97,6 +98,7 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.ERROR); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder() + .manualRedactions(new ManualRedactions()) .messageType(MessageType.FULL_ANALYSE) .analysisVersion(100) .fileId(file.getId()) @@ -132,12 +134,13 @@ public class FileProcessingTest extends AbstractPersistenceServerServiceTest { @Test - public void testFileReanlyseAfterToggleExclusion(){ + public void testFileReanlyseAfterToggleExclusion() { var dossier = dossierTesterAndProvider.provideTestDossier(); var file = fileTesterAndProvider.testAndProvideFile(dossier); fileProcessingClient.analysisSuccessful(dossier.getId(), file.getId(), AnalyzeResult.builder() + .manualRedactions(new ManualRedactions()) .messageType(MessageType.FULL_ANALYSE) .analysisVersion(100) .fileId(file.getId()) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 3b5dfc47a..94736dc06 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -291,8 +291,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .build()); var loadedForceRedaction2 = manualRedactionClient.getForceRedaction(file.getId(), forceRedaction2.getAnnotationId()); assertThat(loadedForceRedaction2.getStatus()).isEqualTo(AnnotationStatus.APPROVED); - assertThat(loadedForceRedaction2.getProcessedDate()).isNotNull(); - + assertThat(loadedForceRedaction2.getProcessedDate()).isNull(); manualRedactionClient.updateForceRedactionStatus(dossier.getId(), file.getId(), forceRedaction2.getAnnotationId(), JSONPrimitive.of(AnnotationStatus.REQUESTED)); loadedForceRedaction2 = manualRedactionClient.getForceRedaction(file.getId(), forceRedaction2.getAnnotationId()); From 19b8bb7aa7a2e0e28f3e856d8b00bd81a7734e7c Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 27 Jan 2022 11:18:21 +0200 Subject: [PATCH 3/4] Fixded PMD --- .../v1/server/service/ExcludeFromAnalysisService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java index ad252f754..f47d53df9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java @@ -37,7 +37,7 @@ public class ExcludeFromAnalysisService { // toggle status fileStatusService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis); - if(!excludedFromAutomaticAnalysis){ + if (!excludedFromAutomaticAnalysis) { // if file has been re-enabled - process it fileStatusService.setStatusFullReprocess(dossierId, fileId, 2); } From a2066533e286e8a92c9900a15aa1430c40fe81a2 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 27 Jan 2022 11:19:57 +0200 Subject: [PATCH 4/4] Fixded PMD --- .../v1/server/service/ExcludeFromAnalysisService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java index f47d53df9..2d26e0ed9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ExcludeFromAnalysisService.java @@ -32,8 +32,6 @@ public class ExcludeFromAnalysisService { public void toggleAutomaticAnalysis(String dossierId, String fileId, boolean excludedFromAutomaticAnalysis) { - var status = fileStatusService.getStatus(fileId); - // toggle status fileStatusService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis);