diff --git a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts index ce50d8ab3..f2126af71 100644 --- a/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts @@ -10,6 +10,7 @@ val redactionServiceVersion by rootProject.extra { "4.290.0" } val pdftronRedactionServiceVersion by rootProject.extra { "4.90.0-RED10115.0" } val redactionReportServiceVersion by rootProject.extra { "4.81.0" } val searchServiceVersion by rootProject.extra { "2.90.0" } +val documentVersion by rootProject.extra { "4.433.0" } repositories { mavenLocal() diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 7dd1054f8..a9640eec9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -14,6 +14,7 @@ configurations { } } dependencies { + api(project(":persistence-service-shared-api-v1")) api(project(":persistence-service-shared-mongo-v1")) api(project(":persistence-service-external-api-v1")) @@ -70,7 +71,6 @@ dependencies { api("commons-validator:commons-validator:1.7") api("com.opencsv:opencsv:5.9") - implementation("com.google.protobuf:protobuf-java:4.27.1") implementation("org.mapstruct:mapstruct:1.6.2") annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntity.java new file mode 100644 index 000000000..f8e3c5bcd --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntity.java @@ -0,0 +1,39 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; + +import java.time.OffsetDateTime; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.IdClass; +import jakarta.persistence.Table; +import jakarta.validation.constraints.NotNull; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Entity +@Builder +@AllArgsConstructor +@NoArgsConstructor +@IdClass(AppVersionEntityKey.class) +@Table(name = "app_version_history") +@FieldDefaults(level = AccessLevel.PRIVATE) +public class AppVersionEntity { + + @Id + @Column + @NotNull String appVersion; + + @Id + @Column + @NotNull String layoutParserVersion; + + @Column + @NotNull OffsetDateTime date; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntityKey.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntityKey.java new file mode 100644 index 000000000..02231f918 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/AppVersionEntityKey.java @@ -0,0 +1,27 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; + +import java.io.Serializable; + +import jakarta.persistence.Column; +import jakarta.validation.constraints.NotNull; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class AppVersionEntityKey implements Serializable { + + @Column + @NotNull String appVersion; + + @Column + @NotNull String layoutParserVersion; + +} + + diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index eeb3c3246..188f52130 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -77,6 +77,9 @@ public class FileEntity { @Column private OffsetDateTime lastLayoutProcessed; + @Column + private String layoutParserVersion; + @Column private OffsetDateTime lastIndexed; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/AppVersionTracker.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/AppVersionTracker.java new file mode 100644 index 000000000..f2d00cfca --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/lifecycle/AppVersionTracker.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.persistence.management.v1.processor.lifecycle; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AppVersionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.TenantProvider; + +import jakarta.validation.ConstraintViolationException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Component +@RequiredArgsConstructor +public class AppVersionTracker { + + @Value("${BACKEND_APP_VERSION:}") + private String appVersion; + + @Value("${LAYOUT_PARSER_VERSION:}") + private String layoutParserVersion; + + private final AppVersionPersistenceService appVersionPersistenceService; + private final TenantProvider tenantProvider; + + + @EventListener(ApplicationReadyEvent.class) + public void trackAppVersion() { + + tenantProvider.getTenants() + .forEach(tenant -> { + + if (!TenantUtils.isTenantReadyForPersistence(tenant)) { + return; + } + + TenantContext.setTenantId(tenant.getTenantId()); + + try { + if (appVersionPersistenceService.insertIfNotExists(appVersion, layoutParserVersion)) { + log.info("Started with new app version {} / layout parser version {}.", appVersion, layoutParserVersion); + } + } catch (ConstraintViolationException cve) { + log.error("Validation failed for app version {} / layout parser version {}.", appVersion, layoutParserVersion, cve); + } catch (Exception e) { + log.error("Failed to track app version {} / layout parser version {}.", appVersion, layoutParserVersion, e); + } + }); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index 9029f4ff7..cc4bed78b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.management.v1.processor.service; -import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure; import java.io.BufferedInputStream; import java.io.File; @@ -18,15 +17,15 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.StorageId import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; +import com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentStructureWrapper; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.RequiredArgsConstructor; @@ -321,7 +320,7 @@ public class FileManagementStorageService { return new DocumentStructureWrapper(storageService.readProtoObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), - DocumentStructure.parser())); + DocumentStructureProto.DocumentStructure.parser())); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java index 9e5aaf0c3..71d06286c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java @@ -50,6 +50,7 @@ public class FileStatusMapper { .legalBasisVersion(status.getLegalBasisVersion()) .lastProcessed(status.getLastProcessed()) .lastLayoutProcessed(status.getLastLayoutProcessed()) + .layoutParserVersion(status.getLayoutParserVersion()) .approvalDate(status.getApprovalDate()) .lastUploaded(status.getLastUploaded()) .analysisDuration(status.getAnalysisDuration()) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 7257e27ee..9284cc37e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -1079,9 +1079,9 @@ public class FileStatusService { } - public void updateLayoutProcessedTime(String fileId) { + public void updateLayoutParserVersionAndProcessedTime(String fileId, String version) { - fileStatusPersistenceService.updateLayoutProcessedTime(fileId); + fileStatusPersistenceService.updateLayoutParserVersionAndProcessedTime(fileId, version); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java index cc2b6ca33..731cc6e63 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java @@ -12,8 +12,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType; +import com.iqser.red.service.redaction.v1.server.data.DocumentStructureWrapper; +import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -35,7 +35,7 @@ public class ImageSimilarityService { DocumentStructureWrapper documentStructure = fileManagementStorageService.getDocumentStructure(dossierId, fileId); List imageDocuments = new ArrayList<>(); documentStructure.streamAllEntries() - .filter(entry -> entry.getType().equals(NodeType.IMAGE)) + .filter(entry -> entry.getType().equals(NodeTypeProto.NodeType.IMAGE)) .forEach(i -> { Map properties = i.getPropertiesMap(); ImageDocument imageDocument = new ImageDocument(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DocumentDataFilesConversionJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DocumentDataFilesConversionJob.java index c0723cac9..e0a3690f6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DocumentDataFilesConversionJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DocumentDataFilesConversionJob.java @@ -21,24 +21,23 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.redaction.v1.server.data.DocumentPageProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto; +import com.iqser.red.service.redaction.v1.server.data.EntryDataProto; +import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto; +import com.iqser.red.service.redaction.v1.server.data.NodeTypeProto; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentPage; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentPositionData; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentStructure; +import com.iqser.red.service.redaction.v1.server.data.old.DocumentTextData; +import com.iqser.red.service.redaction.v1.server.data.old.LayoutEngine; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantProvider; -import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/AppVersionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/AppVersionPersistenceService.java new file mode 100644 index 000000000..3e869b70f --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/AppVersionPersistenceService.java @@ -0,0 +1,57 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence; + +import java.time.OffsetDateTime; +import java.util.Optional; + +import org.springframework.stereotype.Service; +import org.springframework.validation.annotation.Validated; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.AppVersionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.AppVersionEntityKey; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AppVersionRepository; + +import jakarta.transaction.Transactional; +import jakarta.validation.ConstraintViolationException; +import jakarta.validation.constraints.NotBlank; +import lombok.RequiredArgsConstructor; + +@Service +@Validated +@RequiredArgsConstructor +public class AppVersionPersistenceService { + + private final AppVersionRepository appVersionRepository; + + + /** + * Inserts a new AppVersionEntity if it does not already exist in the repository. + * + * @param appVersion the version of the application, must not be blank + * @param layoutParserVersion the version of the layout parser, must not be blank + * @return {@code true} if the entity was inserted; {@code false} if it already exists + * @throws ConstraintViolationException if the input parameters fail validation + */ + @Transactional + public boolean insertIfNotExists(@NotBlank String appVersion, @NotBlank String layoutParserVersion) throws ConstraintViolationException { + + AppVersionEntityKey key = new AppVersionEntityKey(appVersion, layoutParserVersion); + if (!appVersionRepository.existsById(key)) { + AppVersionEntity newAppVersion = new AppVersionEntity(appVersion, layoutParserVersion, OffsetDateTime.now()); + appVersionRepository.save(newAppVersion); + return true; + } + return false; + } + + + /** + * Retrieves the latest AppVersionEntity based on the date field. + * + * @return an Optional containing the latest AppVersionEntity if present, otherwise empty + */ + public Optional getLatestAppVersion() { + + return appVersionRepository.findTopByOrderByDateDesc(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index fa294214d..7ab237e08 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -692,9 +692,9 @@ public class FileStatusPersistenceService { } - public void updateLayoutProcessedTime(String fileId) { + public void updateLayoutParserVersionAndProcessedTime(String fileId, String version) { - fileRepository.updateLayoutProcessedTime(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + fileRepository.updateLayoutProcessedTime(fileId, version, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/AppVersionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/AppVersionRepository.java new file mode 100644 index 000000000..e6ce2fa52 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/AppVersionRepository.java @@ -0,0 +1,14 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.AppVersionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.AppVersionEntityKey; + +public interface AppVersionRepository extends JpaRepository { + + Optional findTopByOrderByDateDesc(); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 410071c87..20735338c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -14,7 +14,6 @@ import org.springframework.data.repository.query.Param; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierChangeProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileChangeProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; @@ -401,8 +400,8 @@ public interface FileRepository extends JpaRepository { @Transactional @Modifying(clearAutomatically = true) - @Query(value = "update FileEntity f set f.lastLayoutProcessed = :offsetDateTime where f.id = :fileId") - void updateLayoutProcessedTime(@Param("fileId") String fileId, @Param("offsetDateTime") OffsetDateTime offsetDateTime); + @Query(value = "update FileEntity f set f.layoutParserVersion = :version, f.lastLayoutProcessed = :offsetDateTime where f.id = :fileId") + void updateLayoutProcessedTime(@Param("fileId") String fileId, @Param("version") String version, @Param("offsetDateTime") OffsetDateTime offsetDateTime); @Query("select f.filename from FileEntity f where f.id = :fileId") @@ -471,12 +470,12 @@ public interface FileRepository extends JpaRepository { @Query("UPDATE FileEntity f SET f.lastDownload = :lastDownload WHERE f.id IN :fileIds AND f.workflowStatus = 'APPROVED'") void updateLastDownloadForApprovedFiles(@Param("fileIds") List fileIds, @Param("lastDownload") DownloadStatusEntity lastDownload); + @Modifying @Query("UPDATE FileEntity f SET f.lastDownload = :lastDownload WHERE f.id = :fileId") void updateLastDownloadForFile(@Param("fileId") String fileId, @Param("lastDownload") DownloadStatusEntity lastDownload); - @Query("SELECT f FROM FileEntity f WHERE f.id in :fileIds AND f.dossierId = :dossierId") List findAllDossierIdAndIds(@Param("dossierId") String dossierId, @Param("fileIds") Set fileIds); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index c10a40bf7..a347033e4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -59,18 +59,16 @@ public class LayoutParsingFinishedMessageReceiver { return; } - var templateId = ""; - var storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); - fileStatusService.setStatusAnalyse(QueueMessageIdentifierService.parseDossierId(response.identifier()), QueueMessageIdentifierService.parseFileId(response.identifier()), QueueMessageIdentifierService.parsePriority(response.identifier())); - fileStatusService.updateLayoutProcessedTime(QueueMessageIdentifierService.parseFileId(response.identifier())); + fileStatusService.updateLayoutParserVersionAndProcessedTime(QueueMessageIdentifierService.parseFileId(response.identifier()), response.layoutParserVersion()); + websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.LAYOUT_UPDATE, fileStatusService.getStatus(fileId).getNumberOfAnalyses() + 1); try { - imageSimilarityService.saveImages(templateId, dossierId, fileId); + imageSimilarityService.saveImages("", dossierId, fileId); } catch (Exception e) { log.error("Error occured during save images: {} ", e.getMessage(), e); throw e; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 3e6316843..017d13960 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -255,3 +255,5 @@ databaseChangeLog: file: db/changelog/tenant/156-add-last-download-to-file.yaml - include: file: db/changelog/tenant/157-add-included-to-csv-export-field.yaml + - include: + file: db/changelog/tenant/158-add-app-version-history-table-and-layout-parser-version-field-to-file.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/158-add-app-version-history-table-and-layout-parser-version-field-to-file.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/158-add-app-version-history-table-and-layout-parser-version-field-to-file.yaml new file mode 100644 index 000000000..0aaf844db --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/158-add-app-version-history-table-and-layout-parser-version-field-to-file.yaml @@ -0,0 +1,37 @@ +databaseChangeLog: + - changeSet: + id: add-app-version-history-table + author: maverick + changes: + - createTable: + tableName: app_version_history + columns: + - column: + name: app_version + type: VARCHAR(128) + constraints: + nullable: false + primaryKey: true + primaryKeyName: app_version_history_pkey + - column: + name: layout_parser_version + type: VARCHAR(128) + constraints: + nullable: false + primaryKey: true + primaryKeyName: app_version_history_pkey + - column: + name: date + type: TIMESTAMP WITHOUT TIME ZONE + constraints: + nullable: false + - changeSet: + id: add-layout-parser-version-to-file + author: maverick + changes: + - addColumn: + tableName: file + columns: + - column: + name: layout_parser_version + type: VARCHAR(128) diff --git a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts index ae7f8c2e2..bf80e5a5a 100644 --- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts @@ -35,7 +35,6 @@ dependencies { testImplementation("org.testcontainers:postgresql:1.17.1") testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4") testImplementation("com.yannbriancon:spring-hibernate-query-utils:2.0.0") - testImplementation("com.google.protobuf:protobuf-java:4.27.1") } description = "persistence-service-server-v1" diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 8ca1a6add..749c2b278 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -97,10 +97,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Lega import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.NotificationType; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentPageProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentPositionDataProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentStructureProto; +import com.iqser.red.service.redaction.v1.server.data.DocumentTextDataProto; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.model.TenantResponse; diff --git a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts index cddbf678d..3b65aa3cc 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts @@ -7,7 +7,10 @@ val springBootStarterVersion = "3.1.5" dependencies { - api("com.knecon.fforesight:layoutparser-service-internal-api:0.188.0") { + api("com.knecon.fforesight:document:${rootProject.extra.get("documentVersion")}"){ + exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") + } + api("com.knecon.fforesight:layoutparser-service-internal-api:0.194.0-RED9998.1") { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java index 3aad6ecf8..5598d25a2 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java @@ -112,6 +112,8 @@ public class FileStatus { private OffsetDateTime lastProcessed; @Schema(description = "Shows the last date of a layout parsing.") private OffsetDateTime lastLayoutProcessed; + @Schema(description = "Shows the version of the layout parser used for the last processing.") + private String layoutParserVersion; @Schema(description = "Shows the date of approval, if approved.") private OffsetDateTime approvalDate; @Schema(description = "Shows last date the document was uploaded.") diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index cedb51147..bdadfabad 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -32,6 +32,7 @@ public class FileModel { private OffsetDateTime lastProcessed; private OffsetDateTime lastIndexed; private OffsetDateTime lastLayoutProcessed; + private String layoutParserVersion; private OffsetDateTime lastManualChangeDate; private int numberOfAnalyses; private String assignee;