From f27b1ab679c1cb07b4efa1b8f9a0b918dd5ec2ef Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 12 Jan 2022 14:13:20 +0200 Subject: [PATCH 1/4] changes to dossier changes endpoint --- .../dossier/DossierChange.java | 15 ++++++++++ .../v1/api/resources/DossierResource.java | 5 ++-- .../DossierPersistenceService.java | 29 +++++++++++++++++-- .../repository/DossierRepository.java | 6 +++- .../repository/FileRepository.java | 5 ++-- .../server/controller/DossierController.java | 9 ++---- .../v1/server/service/DossierService.java | 4 ++- .../server/integration/tests/DossierTest.java | 7 +++++ .../v1/server/integration/tests/FileTest.java | 7 +++++ 9 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierChange.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierChange.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierChange.java new file mode 100644 index 000000000..8d05eb136 --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierChange.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier; + +import lombok.*; + +@Data +@Builder +@EqualsAndHashCode(of="dossierId") +@NoArgsConstructor +@AllArgsConstructor +public class DossierChange { + + private String dossierId; + private boolean dossierChanges; + private boolean fileChanges; +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierResource.java index fc4b05f7a..a8b1d891f 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierResource.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.resources; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierInformation; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -26,12 +27,12 @@ public interface DossierResource { String INCLUDE_DELETED_PARAM = "includeDeleted"; - String CHANGES_PATH = "/has-changes"; + String CHANGES_PATH = "/changes"; @ResponseBody @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = REST_PATH + CHANGES_PATH, produces = MediaType.APPLICATION_JSON_VALUE) - JSONPrimitive hasChangesSince(@RequestBody JSONPrimitive since); + Set changesSince(@RequestBody JSONPrimitive since); @PostMapping(value = REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) Dossier addDossier(@RequestBody CreateOrUpdateDossierRequest dossierRequest); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index fadf814db..dc7acb5d6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -8,6 +8,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; import lombok.RequiredArgsConstructor; import org.springframework.beans.BeanUtils; @@ -16,7 +17,9 @@ import org.springframework.stereotype.Service; import javax.transaction.Transactional; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -62,7 +65,11 @@ public class DossierPersistenceService { dossier.getReportTemplates().forEach(t -> t.getDossiers().removeIf(d -> d.getId().equals(dossierId) && !createOrUpdateDossierRequest.getReportTemplateIds().contains(t.getTemplateId()))); var reportTemplates = reportTemplateRepository.findAllById(createOrUpdateDossierRequest.getReportTemplateIds()); - reportTemplates.forEach(r -> {if (!r.getDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()).contains(dossierId)) {r.getDossiers().add(dossier);}}); + reportTemplates.forEach(r -> { + if (!r.getDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()).contains(dossierId)) { + r.getDossiers().add(dossier); + } + }); dossier.setReportTemplates(reportTemplates); }); @@ -110,7 +117,23 @@ public class DossierPersistenceService { dossierRepository.markDossierAsDeleted(dossierId, DossierStatus.DELETED, softDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } - public boolean hasChangesSince(OffsetDateTime since) { - return dossierRepository.existsByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS)) || fileRepository.existsByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS)); + public Set hasChangesSince(OffsetDateTime since) { + + var dossiersWithChanges = dossierRepository.findDossierChangeByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS)); + var dossiersWithChangedFiles = fileRepository.findDossierChangeByLastUpdatedIsAfter(since.truncatedTo(ChronoUnit.MILLIS)); + + Set dossierChanges = new HashSet<>(); + + for (var dossierId : dossiersWithChanges) { + var dossierChange = new DossierChange(dossierId, true, dossiersWithChangedFiles.contains(dossierId)); + dossierChanges.add(dossierChange); + } + + for (var dossierId : dossiersWithChangedFiles) { + var dossierChange = new DossierChange(dossierId, dossiersWithChanges.contains(dossierId), true); + dossierChanges.add(dossierChange); + } + + return dossierChanges; } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index d0da0ce50..23ea14f2b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -1,15 +1,20 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import java.time.OffsetDateTime; +import java.util.List; public interface DossierRepository extends JpaRepository { + @Query("select d.id from DossierEntity d where d.lastUpdated > :since") + List findDossierChangeByLastUpdatedIsAfter(OffsetDateTime since); + @Modifying @Query("update DossierEntity d set d.status = :dossierStatus, d.hardDeletedTime = :hardDeletedTime, d.lastUpdated = :lastUpdated," + " d.softDeletedTime = " + @@ -29,6 +34,5 @@ public interface DossierRepository extends JpaRepository @Query("update DossierEntity d set d.status = :dossierStatus, d.softDeletedTime = :softDeletedTime, d.lastUpdated = :lastUpdated where d.id = :dossierId") void markDossierAsDeleted(String dossierId, DossierStatus dossierStatus, OffsetDateTime softDeletedTime, OffsetDateTime lastUpdated); - boolean existsByLastUpdatedIsAfter(OffsetDateTime since); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 63adde14e..8ed3e2697 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import org.springframework.data.jpa.repository.JpaRepository; @@ -16,8 +17,6 @@ public interface FileRepository extends JpaRepository { boolean existsByDossierIdAndLastUpdatedIsAfter(String dossierId, OffsetDateTime since); - boolean existsByLastUpdatedIsAfter(OffsetDateTime since); - List findByDossierId(String dossierId); @Modifying @@ -121,6 +120,8 @@ public interface FileRepository extends JpaRepository { @Query("select count(f) from FileEntity f where f.dossierId = :dossierId and f.deleted is not null and f.hardDeletedTime is null") int countSoftDeletedFiles(String dossierId); + @Query("select distinct f.dossierId from FileEntity f where f.lastUpdated > :since") + List findDossierChangeByLastUpdatedIsAfter(OffsetDateTime since); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java index f2c144543..095fcc441 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java @@ -8,10 +8,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierInformation; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.*; import com.iqser.red.service.persistence.service.v1.api.resources.DossierResource; import feign.Param; import lombok.RequiredArgsConstructor; @@ -39,9 +36,9 @@ public class DossierController implements DossierResource { @Override - public JSONPrimitive hasChangesSince(@RequestBody JSONPrimitive since) { + public Set changesSince(@RequestBody JSONPrimitive since) { - return JSONPrimitive.of(dossierService.hasChangesSince(since.getValue())); + return dossierService.changesSince(since.getValue()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierService.java index 613758efb..b47aeaa73 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierService.java @@ -8,12 +8,14 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.time.OffsetDateTime; import java.util.List; +import java.util.Set; /** * Provides the internal interface between dossier request and the actual persistence. @@ -108,7 +110,7 @@ public class DossierService { return dossierPersistenceService.findAllDossiers(); } - public boolean hasChangesSince(OffsetDateTime since) { + public Set changesSince(OffsetDateTime since) { return dossierPersistenceService.hasChangesSince(since); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java index 919ca1ee2..3e30ca6c5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTest.java @@ -3,8 +3,10 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.time.OffsetDateTime; import java.util.List; +import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import org.junit.Test; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -35,12 +37,17 @@ public class DossierTest extends AbstractPersistenceServerServiceTest { @Test public void testDossier() { + var start = OffsetDateTime.now(); + var dossier = dossierTesterAndProvider.provideTestDossier(); var allDossiers = dossierClient.getAllDossiers(); assertThat(allDossiers.size()).isEqualTo(1); assertThat(allDossiers.get(0)).isEqualTo(dossier); + var changes = dossierClient.changesSince(JSONPrimitive.of(start)); + assertThat(changes).isNotEmpty(); + // update CreateOrUpdateDossierRequest cru = new CreateOrUpdateDossierRequest(); cru.setDossierTemplateId(dossier.getId()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index a26ade59d..c984fa358 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.Collections; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; @@ -101,6 +102,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Test public void testFile() { + var start = OffsetDateTime.now(); var dossier = dossierTesterAndProvider.provideTestDossier(); // update @@ -111,6 +113,11 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier); + var changes = dossierClient.changesSince(JSONPrimitive.of(start)); + assertThat(changes).isNotEmpty(); + assertThat(changes.iterator().next().isFileChanges()).isTrue(); + assertThat(changes.iterator().next().isDossierChanges()).isTrue(); + assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); From 36353463c935ceed8469bcfdcd3b5298b2744f71 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 13 Jan 2022 09:25:24 +0200 Subject: [PATCH 2/4] analysis version in redactionlog and changes --- .../FileStatusPersistenceService.java | 4 +- .../repository/FileRepository.java | 6 +- .../v1/server/migration/MigrationService.java | 142 ++++-------------- .../v1/server/service/FileStatusService.java | 7 +- .../service/FileTesterAndProvider.java | 3 +- .../AbstractPersistenceServerServiceTest.java | 3 +- persistence-service-v1/pom.xml | 2 +- 7 files changed, 45 insertions(+), 122 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 56260af9d..ad37b0535 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -53,14 +53,14 @@ public class FileStatusPersistenceService { @Transactional public void updateWorkflowStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long duration, long dossierDictionaryVersion, - int analysisVersion) { + int analysisVersion, int analysisNumber) { if (isFileDeleted(fileId)) { return; } fileRepository.updateWorkflowStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 8ed3e2697..9e4754715 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -38,12 +38,12 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + - "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = f.numberOfAnalyses + 1, f.lastUpdated = :lastUpdated, " + + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed " + "where f.id = :fileId") void updateWorkflowStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion, - int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed); + int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber); @Modifying @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + @@ -109,7 +109,7 @@ public interface FileRepository extends JpaRepository { @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + - "f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, " + + "f.assignee = null, f.approvalDate = null, f.lastManualRedaction = null, f.numberOfAnalyses = 0, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null " + diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java index 0b34add1f..9f1dc2a6d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java @@ -1,127 +1,42 @@ package com.iqser.red.service.peristence.v1.server.migration; -import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; -import static java.util.stream.Collectors.toList; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.peristence.v1.server.exception.MigrationException; -import com.iqser.red.service.peristence.v1.server.migration.model.CommentRow; -import com.iqser.red.service.peristence.v1.server.migration.model.ConfigurationRow; -import com.iqser.red.service.peristence.v1.server.migration.model.DigitalSignatureModel; -import com.iqser.red.service.peristence.v1.server.migration.model.DossierAttributeRow; -import com.iqser.red.service.peristence.v1.server.migration.model.DossierAttributesConfig; -import com.iqser.red.service.peristence.v1.server.migration.model.EntryRow; -import com.iqser.red.service.peristence.v1.server.migration.model.FileAttributesConfig; -import com.iqser.red.service.peristence.v1.server.migration.model.IdRemovalRow; -import com.iqser.red.service.peristence.v1.server.migration.model.ManualForceRedactionRow; -import com.iqser.red.service.peristence.v1.server.migration.model.ManualImageRecategorizationRow; -import com.iqser.red.service.peristence.v1.server.migration.model.ManualLegalBasisChangeRow; -import com.iqser.red.service.peristence.v1.server.migration.model.ManualRedactionEntryRow; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationAuditModel; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationComment; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDossier; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDossierTemplate; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationDownloadStatus; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationFileStatus; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationNotification; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationRedactionLog; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationRedactionLogEntry; -import com.iqser.red.service.peristence.v1.server.migration.model.MigrationViewedPage; -import com.iqser.red.service.peristence.v1.server.migration.model.NewIdForDossierTemplate; -import com.iqser.red.service.peristence.v1.server.migration.model.Status; -import com.iqser.red.service.peristence.v1.server.migration.model.VersionRow; -import com.iqser.red.service.peristence.v1.server.migration.model.VersionType; +import com.iqser.red.service.peristence.v1.server.migration.model.*; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualImageRecategorizationEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.*; import com.iqser.red.service.persistence.management.v1.processor.entity.audit.AuditEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DigitalSignatureEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.LegalBasisMappingEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.RuleSetEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.*; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.service.EncryptionDecryptionService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ForceRedactionRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ImageRecategorizationRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisChangeRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ManualRedactionRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.redaction.v1.model.Change; -import com.iqser.red.service.redaction.v1.model.ChangeType; -import com.iqser.red.service.redaction.v1.model.Engine; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; +import static java.util.stream.Collectors.toList; @SuppressWarnings("PMD") @Slf4j @@ -264,41 +179,42 @@ public class MigrationService { List files = fileRepository.findAll(); files.stream().filter(file -> file.getHardDeletedTime() == null).forEach(file -> { - migrateRedactionLog(file.getDossierId(), file.getId(), file.getLastProcessed()); + migrateRedactionLog(file); }); } @SneakyThrows - private void migrateRedactionLog(String dossierId, String fileId, OffsetDateTime lastProcessed) { + private void migrateRedactionLog(FileEntity file) { try { - byte[] oldRedactionLogBytes = fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG)); + byte[] oldRedactionLogBytes = fileManagementStorageService.getStoredObjectBytes(StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG)); if (oldRedactionLogBytes == null) { - log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", dossierId, fileId); + log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", file.getDossierId(), file.getId()); return; } var oldRedactionLog = objectMapper.readValue(oldRedactionLogBytes, MigrationRedactionLog.class); - RedactionLog redactionLog = new RedactionLog(oldRedactionLog.getAnalysisVersion(), oldRedactionLog.getRedactionLogEntry() + RedactionLog redactionLog = new RedactionLog(oldRedactionLog.getAnalysisVersion(), + file.getNumberOfAnalyses(), oldRedactionLog.getRedactionLogEntry() .stream() - .map(r -> convertRedactionLogEntry(r, fileId, lastProcessed)) + .map(r -> convertRedactionLogEntry(r, file.getId(), file.getLastProcessed(),file.getNumberOfAnalyses())) .collect(toList()), oldRedactionLog.getLegalBasis(), oldRedactionLog.getDictionaryVersion(), oldRedactionLog .getDossierDictionaryVersion(), oldRedactionLog.getRulesVersion(), oldRedactionLog.getLegalBasisVersion()); - storageService.storeObject("backup/" + StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), objectMapper + storageService.storeObject("backup/" + StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG), objectMapper .writeValueAsBytes(oldRedactionLogBytes)); - storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), objectMapper + storageService.storeObject(StorageIdUtils.getStorageId(file.getDossierId(), file.getId(), FileType.REDACTION_LOG), objectMapper .writeValueAsBytes(redactionLog)); } catch (StorageObjectDoesNotExist e) { - log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", dossierId, fileId); + log.warn("Not redactionLog found to convert for dossierId {} and fileId {}", file.getDossierId(), file.getId()); } } private RedactionLogEntry convertRedactionLogEntry(MigrationRedactionLogEntry oldRedactionLogEntry, String fileId, - OffsetDateTime lastProcessed) { + OffsetDateTime lastProcessed, int analysisNumber) { return RedactionLogEntry.builder() .id(oldRedactionLogEntry.getId()) @@ -335,7 +251,7 @@ public class MigrationService { .excluded(oldRedactionLogEntry.isExcluded()) .recategorizationType(oldRedactionLogEntry.getRecategorizationType()) .legalBasisChangeValue(oldRedactionLogEntry.getLegalBasisChangeValue()) - .changes(List.of(new Change(ChangeType.ADDED, lastProcessed))) + .changes(List.of(new Change(analysisNumber, ChangeType.ADDED, lastProcessed))) .engines(getEngine(oldRedactionLogEntry)) .reference(null) .rectangle(oldRedactionLogEntry.isRectangle()) @@ -346,7 +262,7 @@ public class MigrationService { private Set getEngine(MigrationRedactionLogEntry oldRedactionLogEntry) { if (oldRedactionLogEntry.isManual()) { - return new HashSet(); + return new HashSet<>(); } if (oldRedactionLogEntry.isDictionaryEntry() || oldRedactionLogEntry.isDossierDictionaryEntry()) { return Set.of(Engine.DICTIONARY); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index fc7bd5e39..2d7bc557a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -101,7 +101,11 @@ public class FileStatusService { public void setStatusSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) { - fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion()); + fileStatusPersistenceService.updateWorkflowStatus(fileId, analyzeResult.getNumberOfPages(), + analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), + analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), + analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), + analyzeResult.getAnalysisNumber()); } @@ -267,6 +271,7 @@ public class FileStatusService { var analyseRequest = AnalyzeRequest.builder() .messageType(reanalyse ? MessageType.REANALYSE : MessageType.FULL_ANALYSE) .dossierId(dossierId) + .analysisNumber(fileStatus.getNumberOfAnalyses()+1) .sectionsToReanalyse(sectionsToReanalyse) .fileId(fileId) .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index dedfb9f3b..6967ad213 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -58,7 +58,8 @@ public class FileTesterAndProvider { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1); - fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()), + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(new RedactionLog(1, 1, + List.of(RedactionLogEntry.builder().id("annotationId").type("manual").value("value entry").build()), null, 0, 0, 0, 0))); fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.SECTION_GRID, objectMapper.writeValueAsBytes(new SectionGrid())); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index fefd91b76..ce7aa679a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -129,7 +129,8 @@ public abstract class AbstractPersistenceServerServiceTest { when(redactionClient.annotate(Mockito.any())).thenReturn(new AnnotateResponse("document".getBytes())); when(redactionClient.sections(Mockito.any())).thenReturn(new RedactionResult()); - when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, Lists.newArrayList(), null, 0, 0, 0, 0)); + when(redactionClient.getRedactionLog(Mockito.any())).thenReturn(new RedactionLog(1, 1, + Lists.newArrayList(), null, 0, 0, 0, 0)); } @After diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index cfabcc5de..a974f1426 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -25,7 +25,7 @@ - 3.43.0 + 3.54.2 2.10.0 3.17.0 3.19.0 From 7ff197d73029a200aea64e7898d28763cbd71373 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 13 Jan 2022 10:59:09 +0200 Subject: [PATCH 3/4] analysis version in redactionlog and changes --- persistence-service-v1/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index a974f1426..f238c0672 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -25,7 +25,7 @@ - 3.54.2 + 3.56.0 2.10.0 3.17.0 3.19.0 From d0aba87bce610ac423501a39caf6610fd0a5907d Mon Sep 17 00:00:00 2001 From: aoezyetimoglu Date: Thu, 13 Jan 2022 15:26:33 +0100 Subject: [PATCH 4/4] RED-3183: Upgrade log4j to 2.17.1 --- persistence-service-v1/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index f238c0672..c2903f2bb 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -6,7 +6,7 @@ com.iqser.red platform-dependency - 1.5.0 + 1.6.0