Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'master' into feature/RED-10342

Browse Source 
# Conflicts:
#	persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
This commit is contained in:
corinaolariu 2024-12-05 10:58:10 +02:00
commit e3eff19de4
30 changed files with 505 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java vendored
View File

@ -16,8 +16,10 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.DossierController;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
import com.iqser.red.persistence.service.v2.external.api.impl.mapper.ComponentMapper;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
@ -29,6 +31,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.users.m
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v2.api.external.model.BulkComponentsRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentOverrideList;
import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
@ -37,6 +40,7 @@ import com.iqser.red.service.persistence.service.v2.api.external.resource.Compon
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -51,11 +55,16 @@ public class ComponentControllerV2 implements ComponentResource {
private final UserService userService;
private final StatusController statusController;
private final FileStatusService fileStatusService;
private final DossierController dossierController;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final CurrentApplicationTypeProvider currentApplicationTypeProvider;
private final ComponentMapper componentMapper = ComponentMapper.INSTANCE;
@Value("${documine.components.filesLimit:100}")
private int documineComponentsFilesLimit = 100;
@Override
public FileComponents getComponents(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@ -92,12 +101,37 @@ public class ComponentControllerV2 implements ComponentResource {
checkApplicationType();
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
var dossierFiles = statusController.getDossierStatus(dossierId);
if(dossierFiles.size() > documineComponentsFilesLimit) {
throw new BadRequestException(String.format("The dossier you requested components for contains %s files this is above the limit of %s files for this endpoint, please use the POST %s", dossierFiles.size(), documineComponentsFilesLimit, FILE_PATH + BULK_COMPONENTS_PATH));
}
return new FileComponentsList(dossierFiles.stream()
.map(file -> getComponents(dossierTemplateId, dossierId, file.getFileId(), includeDetails))
.toList());
}
@Override
public FileComponentsList getComponentsForFiles(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails,
@RequestBody BulkComponentsRequest bulkComponentsRequest){
if(bulkComponentsRequest.getFileIds().size() > documineComponentsFilesLimit) {
throw new BadRequestException(String.format("You requested components for %s files this is above the limit of %s files for this endpoint, lower the fileIds in the request", bulkComponentsRequest.getFileIds().size(), documineComponentsFilesLimit));
}
checkApplicationType();
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
dossierController.getDossier(dossierId, false, false);
return new FileComponentsList(bulkComponentsRequest.getFileIds().stream()
.map(fileId -> getComponents(dossierTemplateId, dossierId, fileId, includeDetails))
.toList());
}
@Override
@PreAuthorize("hasAuthority('" + GET_RSS + "')")
public void addOverride(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,

6
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java vendored
View File

@ -450,11 +450,15 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
return new DossierAttributeDefinitionList(dossierAttributeConfigPersistenceService.getDossierAttributes(dossierTemplateId)
.stream()
.map(config -> DossierAttributeDefinition.builder()
.id(config.getId())
.name(config.getLabel())
.type(config.getType())
.reportingPlaceholder(config.getPlaceholder())
.displaySettings(DossierAttributeDefinition.DossierDisplaySettings.builder()
.editable(config.isEditable())
.filterable(config.isFilterable())
.displayedInDossierList(config.isDisplayedInDossierList())
.build())
.build())
.toList());
}

18
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/BulkComponentsRequest.java vendored Normal file
View File

@ -0,0 +1,18 @@
package com.iqser.red.service.persistence.service.v2.api.external.model;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class BulkComponentsRequest {
private List<String> fileIds = new ArrayList<>();
}

13
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/model/DossierAttributeDefinition.java vendored
View File

@ -17,5 +17,18 @@ public class DossierAttributeDefinition {
private String name;
private DossierAttributeType type;
private String reportingPlaceholder;
private DossierDisplaySettings displaySettings;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class DossierDisplaySettings {
private boolean editable;
private boolean filterable;
private boolean displayedInDossierList;
}
}

11
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/ComponentResource.java vendored
View File

@ -20,6 +20,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.BulkComponentsRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentOverrideList;
import com.iqser.red.service.persistence.service.v2.api.external.model.FileComponents;
@ -74,6 +75,16 @@ public interface ComponentResource {
@Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails);
@PostMapping(value = FILE_PATH
+ BULK_COMPONENTS_PATH, produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}, consumes = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Returns the components for all files of a dossier", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
FileComponentsList getComponentsForFiles(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier that contains the file.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
@Parameter(name = INCLUDE_DETAILS_PARAM, description = INCLUDE_DETAILS_DESCRIPTION) @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails,
@RequestBody BulkComponentsRequest bulkComponentsRequest);
@ResponseBody
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@PostMapping(value = FILE_PATH + FILE_ID_PATH_VARIABLE + OVERRIDES_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)

93
persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml
View File

@ -1626,6 +1626,50 @@ paths:
$ref: '#/components/responses/429'
"500":
$ref: '#/components/responses/500'
post:
operationId: getComponentsForFiles
tags:
- 4. Components
summary: Returns the FileComponents for requested files
description: |
This endpoint fetches components for the requested files by its ids. Like individual file components,
these represent various aspects, metadata or content of the files. Entity and component rules define these components based on the file's
content. They can give a *structured view* on a document's text.
To include detailed component information, set the `includeDetails` query parameter to `true`.
parameters:
- $ref: '#/components/parameters/dossierTemplateId'
- $ref: '#/components/parameters/dossierId'
- $ref: '#/components/parameters/includeComponentDetails'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/BulkComponentsRequest'
required: true
responses:
"200":
content:
application/json:
schema:
$ref: '#/components/schemas/FileComponentsList'
application/xml:
schema:
$ref: '#/components/schemas/FileComponentsList'
description: |
Successfully fetched components for all files in the dossier.
"400":
$ref: '#/components/responses/400'
"401":
$ref: '#/components/responses/401'
"403":
$ref: '#/components/responses/403'
"404":
$ref: '#/components/responses/404-dossier'
"429":
$ref: '#/components/responses/429'
"500":
$ref: '#/components/responses/500'
/api/downloads:
get:
operationId: getDownloadStatusList
@ -2773,6 +2817,16 @@ components:
entityRuleId: DEF.13.37
type: another_entity_type
page: 456
BulkComponentsRequest:
type: object
description: Request payload to get components for multiple files.
properties:
fileIds:
type: array
description: A list with unique identifiers of the files for which components should be retrieved.
items:
type: string
description: The unique identifier of a file.
DossierStatusDefinition:
type: object
description: |
@ -2864,6 +2918,8 @@ components:
placeholder follows a specific format convention:
`{{dossier.attribute.<name>}}` while the name is transformed into 'PascalCase' and does not contain
whitespaces. The placeholder is unique in a dossier template.
displaySettings:
$ref: '#/components/schemas/DossierAttributeDisplaySettings'
required:
- name
- type
@ -2872,6 +2928,35 @@ components:
name: "Document Summary"
type: "TEXT"
reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}"
displaySettings:
editable: true
filterable: false
displayedInDossierList: false
DossierAttributeDisplaySettings:
type: object
description: |
Display setting for the user interface. These settings control how the UI handles and presents the dossier attributes.
properties:
editable:
type: boolean
description: |
If `true`, the user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible.
filterable:
type: boolean
description: |
If `true`, the user interfaces add filter options to the dossier list.
displayedInDossierList:
type: boolean
description: |
if `true`, the user interfaces show the values in the dossier list.
required:
- editable
- filterable
- displayedInDossierList
example:
editable: true
filterable: true
displayedInDossierList: false
FileAttributeDefinition:
type: object
description: |
@ -2994,10 +3079,18 @@ components:
name: "Dossier Summary"
type: "TEXT"
reportingPlaceholder: "{{dossier.attribute.DossierSummary}}"
displaySettings:
editable: true
filterable: false
displayedInFileList: false
- id: "23e45678-e90b-12d3-a456-765114174321"
name: "Comment"
type: "TEXT"
reportingPlaceholder: "{{dossier.attribute.Comment}}"
displaySettings:
editable: true
filterable: false
displayedInFileList: false
FileAttributeDefinitionList:
type: object
description: A list of file attribute definitions.

39
persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml
View File

@ -1492,6 +1492,8 @@ components:
placeholder follows a specific format convention:
`{{dossier.attribute.<name>}}` while the name is transformed into 'PascalCase' and does not contain
whitespaces. The placeholder is unique in a dossier template.
displaySettings:
$ref: '#/components/schemas/DossierAttributeDisplaySettings'
required:
- name
- type
@ -1500,6 +1502,35 @@ components:
name: "Document Summary"
type: "TEXT"
reportingPlaceholder: "{{dossier.attribute.DocumentSummary}}"
displaySettings:
editable: true
filterable: false
displayedInDossierList: false
DossierAttributeDisplaySettings:
type: object
description: |
Display setting for the user interface. These settings control how the UI handles and presents the dossier attributes.
properties:
editable:
type: boolean
description: |
If `true`, the user interfaces allow manual editing of the value. Otherwise only importing and setting by rules would be possible.
filterable:
type: boolean
description: |
If `true`, the user interfaces add filter options to the dossier list.
displayedInDossierList:
type: boolean
description: |
if `true`, the user interfaces show the values in the dossier list.
required:
- editable
- filterable
- displayedInDossierList
example:
editable: true
filterable: true
displayedInDossierList: false
FileAttributeDefinition:
type: object
description: |
@ -1622,10 +1653,18 @@ components:
name: "Dossier Summary"
type: "TEXT"
reportingPlaceholder: "{{dossier.attribute.DossierSummary}}"
displaySettings:
editable: true
filterable: false
displayedInFileList: false
- id: "23e45678-e90b-12d3-a456-765114174321"
name: "Comment"
type: "TEXT"
reportingPlaceholder: "{{dossier.attribute.Comment}}"
displaySettings:
editable: true
filterable: false
displayedInFileList: false
FileAttributeDefinitionList:
type: object
description: A list of file attribute definitions.

4
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -36,12 +36,12 @@ dependencies {
}
api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
implementation("com.knecon.fforesight:llm-service-api:1.20.0-RED10072.2")
api("com.knecon.fforesight:jobs-commons:0.12.0")
api("com.knecon.fforesight:jobs-commons:0.13.0")
api("com.iqser.red.commons:storage-commons:2.50.0")
api("com.knecon.fforesight:tenant-commons:0.31.0-RED10196.0") {
exclude(group = "com.iqser.red.commons", module = "storage-commons")
}
api("com.knecon.fforesight:database-tenant-commons:0.30.0") {
api("com.knecon.fforesight:database-tenant-commons:0.31.0") {
exclude(group = "com.knecon.fforesight", module = "tenant-commons")
}
api("com.knecon.fforesight:keycloak-commons:0.30.0") {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeConfigEntity.java
View File

@ -30,6 +30,10 @@ public class DossierAttributeConfigEntity {
@Column
private boolean editable;
@Column
private boolean filterable;
@Column
private boolean displayedInDossierList;
@Column
private String placeholder;
@Column

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -9,6 +9,7 @@ import java.util.Set;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -22,8 +23,10 @@ import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.ForeignKey;
import jakarta.persistence.Id;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
@ -218,6 +221,11 @@ public class FileEntity {
@Column
private boolean protobufMigrationDone;
@ManyToOne(fetch = FetchType.EAGER)
@JoinColumn(name = "last_download", referencedColumnName = "storage_id", foreignKey = @ForeignKey(name = "fk_file_last_download"))
private DownloadStatusEntity lastDownload;
public OffsetDateTime getLastOCRTime() {
return this.ocrStartTime;

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java
View File

@ -43,6 +43,7 @@ import lombok.experimental.FieldDefaults;
public class DownloadStatusEntity {
@Id
@Column(name = "storage_id")
String storageId;
@Column
String uuid;

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java
View File

@ -125,7 +125,7 @@ public class FileStatusManagementService {
if (userId != null) {
assignee = userId;
}
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.NEW);
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
fileStatusService.setAssignee(fileId, assignee);
indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2);
}
@ -139,7 +139,7 @@ public class FileStatusManagementService {
assignee = approverId;
}
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.NEW);
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
fileStatusService.setAssignee(fileId, approverId);
indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2);
}
@ -169,7 +169,7 @@ public class FileStatusManagementService {
throw new BadRequestException("Allowed transition not possible from: " + fileStatus.getWorkflowStatus() + " to status NEW");
}
fileStatusService.setAssignee(fileId, null);
fileStatusService.setStatusSuccessful(fileId, WorkflowStatus.NEW);
fileStatusService.setStatusSuccessful(fileId, WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
}

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java
View File

@ -69,6 +69,7 @@ public class FileStatusMapper {
.fileSize(status.getFileSize())
.fileErrorInfo(status.getFileErrorInfo())
.componentMappingVersions(status.getComponentMappingVersions())
.lastDownload(status.getLastDownloadDate())
.build();
}

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -816,9 +816,13 @@ public class FileStatusService {
}
public void setStatusSuccessful(String fileId, WorkflowStatus workflowStatus) {
public void setStatusSuccessful(String fileId, WorkflowStatus newWorkflowStatus, WorkflowStatus oldWorkflowStatus) {
fileStatusPersistenceService.updateWorkflowStatus(fileId, workflowStatus, false);
fileStatusPersistenceService.updateWorkflowStatus(fileId, newWorkflowStatus, false);
if(oldWorkflowStatus == WorkflowStatus.APPROVED && newWorkflowStatus != WorkflowStatus.APPROVED) {
fileStatusPersistenceService.clearLastDownload(fileId);
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributeConfigPersistenceService.java
View File

@ -72,6 +72,8 @@ public class DossierAttributeConfigPersistenceService {
config.setLabel(dossierAttributeConfig.getLabel());
config.setType(dossierAttributeConfig.getType());
config.setEditable(dossierAttributeConfig.isEditable());
config.setDisplayedInDossierList(dossierAttributeConfig.isDisplayedInDossierList());
config.setFilterable(dossierAttributeConfig.isFilterable());
setPlaceholder(config);
uniqueLabelAndPlaceholder(dossierAttributeConfig);
return dossierAttributeConfigRepository.save(config);

51
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -34,19 +34,47 @@ public class DownloadStatusPersistenceService {
// use this to create a status for export dossier template.
public void createStatus(String userId, String storageId, String filename, String mimeType) {
this.createStatus(userId, storageId, null, filename, mimeType, null, null, null, null);
this.saveDownloadStatus(userId, storageId, null, filename, mimeType, null, null, null, null);
}
@Transactional
public DownloadStatusEntity createStatus(String userId,
String storageId,
DossierEntity dossier,
String filename,
String mimeType,
List<String> fileIds,
Set<DownloadFileType> downloadFileTypes,
List<String> reportTemplateIds,
String redactionPreviewColor) {
String storageId,
DossierEntity dossier,
String filename,
String mimeType,
List<String> fileIds,
Set<DownloadFileType> downloadFileTypes,
List<String> reportTemplateIds,
String redactionPreviewColor) {
DownloadStatusEntity savedDownloadStatus = saveDownloadStatus(userId,
storageId,
dossier,
filename,
mimeType,
fileIds,
downloadFileTypes,
reportTemplateIds,
redactionPreviewColor);
if (fileIds != null && !fileIds.isEmpty()) {
fileRepository.updateLastDownloadForApprovedFiles(fileIds, savedDownloadStatus);
}
return savedDownloadStatus;
}
private DownloadStatusEntity saveDownloadStatus(String userId,
String storageId,
DossierEntity dossier,
String filename,
String mimeType,
List<String> fileIds,
Set<DownloadFileType> downloadFileTypes,
List<String> reportTemplateIds,
String redactionPreviewColor) {
DownloadStatusEntity downloadStatus = new DownloadStatusEntity();
downloadStatus.setUserId(userId);
@ -62,7 +90,7 @@ public class DownloadStatusPersistenceService {
downloadStatus.setRedactionPreviewColor(redactionPreviewColor);
downloadStatus.setStatus(DownloadStatusValue.QUEUED);
return downloadStatusRepository.save(downloadStatus);
return downloadStatusRepository.saveAndFlush(downloadStatus);
}
@ -134,7 +162,8 @@ public class DownloadStatusPersistenceService {
@Transactional
public DownloadStatusEntity getStatusesByUuid(String uuid) {
return downloadStatusRepository.findByUuid(uuid).orElseThrow(() -> new NotFoundException(String.format("DownloadStatus not found for uuid: %s", uuid)));
return downloadStatusRepository.findByUuid(uuid)
.orElseThrow(() -> new NotFoundException(String.format("DownloadStatus not found for uuid: %s", uuid)));
}

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -754,4 +753,11 @@ public class FileStatusPersistenceService {
return fileRepository.findAllByDossierId(dossierId, includeDeleted);
}
@Transactional
public void clearLastDownload(String fileId) {
fileRepository.updateLastDownloadForFile(fileId, null);
}
}

14
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -12,6 +12,7 @@ import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierChangeProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileChangeProjection;
@ -142,8 +143,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.processingErrorCounter = :processingErrorCounter "
+ "where f.dossierId in (select fe.dossierId from FileEntity fe inner join DossierEntity d on d.id = fe.dossierId where d.dossierTemplateId = :dossierTemplateId) and f.processingStatus = 'ERROR'")
void updateErrorCounter(@Param("dossierTemplateId") String dossierTemplateId,
@Param("processingErrorCounter") int processingErrorCounter);
void updateErrorCounter(@Param("dossierTemplateId") String dossierTemplateId, @Param("processingErrorCounter") int processingErrorCounter);
@Modifying
@ -467,6 +467,16 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
List<String> findAllByDossierId(@Param("dossierId") String dossierId, @Param("includeDeleted") boolean includeDeleted);
@Modifying
@Query("UPDATE FileEntity f SET f.lastDownload = :lastDownload WHERE f.id IN :fileIds AND f.workflowStatus = 'APPROVED'")
void updateLastDownloadForApprovedFiles(@Param("fileIds") List<String> fileIds, @Param("lastDownload") DownloadStatusEntity lastDownload);
@Modifying
@Query("UPDATE FileEntity f SET f.lastDownload = :lastDownload WHERE f.id = :fileId")
void updateLastDownloadForFile(@Param("fileId") String fileId, @Param("lastDownload") DownloadStatusEntity lastDownload);
@Query("SELECT f FROM FileEntity f WHERE f.id in :fileIds AND f.dossierId = :dossierId")
List<FileEntity> findAllDossierIdAndIds(@Param("dossierId") String dossierId, @Param("fileIds") Set<String> fileIds);

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java
View File

@ -20,6 +20,9 @@ public class FileModelMapper implements BiConsumer<FileEntity, FileModel> {
.forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()));
fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp(), fileEntity.getErrorCode()));
fileModel.setComponentMappingVersions(getComponentMappingVersions(fileEntity));
if (fileEntity.getLastDownload() != null) {
fileModel.setLastDownloadDate(fileEntity.getLastDownload().getCreationDate());
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -250,4 +250,6 @@ databaseChangeLog:
- include:
file: db/changelog/tenant/153-custom-technical-name-change.yaml
- include:
file: db/changelog/tenant/154-add-included-to-csv-export-field.yaml
file: db/changelog/tenant/154-add-last-download-to-file.yaml
- include:
file: db/changelog/tenant/155-add-displayed-and-filterable-to-dossier-attribute-config.yaml

28
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/154-add-last-download-to-file.yaml Normal file
View File

@ -0,0 +1,28 @@
databaseChangeLog:
- changeSet:
id: add-last-download-to-file
author: maverick
changes:
- addColumn:
tableName: file
columns:
- column:
name: last_download
type: VARCHAR(255)
constraints:
nullable: true
- changeSet:
id: add-fk-last-download
author: maverick
changes:
- addForeignKeyConstraint:
baseTableName: file
baseColumnNames: last_download
constraintName: fk_file_last_download
referencedTableName: download_status
referencedColumnNames: storage_id
onDelete: SET NULL
onUpdate: NO ACTION
deferrable: false
initiallyDeferred: false
validate: true

16
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/155-add-displayed-and-filterable-to-dossier-attribute-config.yaml Normal file
View File

@ -0,0 +1,16 @@
databaseChangeLog:
- changeSet:
id: 155-add-displayed-and-filterable-to-dossier-attribute-config
author: maverick
changes:
- addColumn:
columns:
- column:
name: displayed_in_dossier_list
type: BOOLEAN
defaultValueBoolean: false
- column:
name: filterable
type: BOOLEAN
defaultValueBoolean: false
tableName: dossier_attribute_config

3
persistence-service-v1/persistence-service-server-v1/build.gradle.kts
View File

@ -28,8 +28,7 @@ dependencies {
api("org.apache.logging.log4j:log4j-slf4j-impl:2.20.0")
api("net.logstash.logback:logstash-logback-encoder:7.4")
implementation("ch.qos.logback:logback-classic")
implementation("org.liquibase:liquibase-core:4.30.0") // Needed to be set explicit, otherwise spring dependency management sets it to 4.20.0
implementation("org.apache.commons:commons-lang3:3.13.0") // Needed for liquibase 4.30.0
implementation("org.liquibase:liquibase-core:4.29.2") // Needed to be set explicit, otherwise spring dependency management sets it to 4.20.0
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
testImplementation("org.springframework.security:spring-security-test:6.0.2")

3
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml
View File

@ -9,6 +9,9 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/
logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
documine:
components:
filesLimit: 150
logging.type: ${LOGGING_TYPE:CONSOLE}
kubernetes.namespace: ${NAMESPACE:default}

1
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java
View File

@ -63,6 +63,7 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest
}
@Test
public void testOverrides() throws IOException {

161
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
View File

@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -57,6 +58,7 @@ import org.junit.jupiter.api.Test;
public class DownloadPreparationTest extends AbstractPersistenceServerServiceTest {
protected static final String USER_ID = "1";
@Autowired
DownloadReportMessageReceiver downloadReportMessageReceiver;
@ -87,8 +89,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
@Autowired
FileClient fileClient;
DossierWithSingleFile testData;
@Autowired
DownloadReadyJob downloadReadyJob;
@ -98,14 +98,13 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
@Autowired
DownloadCompressionMessageReceiver downloadCompressionMessageReceiver;
DossierWithSingleFile testData;
@BeforeEach
public void createTestData() {
testData = new DossierWithSingleFile();
}
@Test
@SneakyThrows
public void testReceiveDownloadPackage() {
@ -159,7 +158,7 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null); // Will be called by scheduler in prod.
var firstStatus = getFirstStatus();
assertThat(getFirstStatus().getStatus()).isEqualTo(DownloadStatusValue.COMPRESSING);
assertThat(firstStatus.getStatus()).isEqualTo(DownloadStatusValue.COMPRESSING);
downloadCompressionMessageReceiver.receive(DownloadJob.builder().storageId(firstStatus.getStorageId()).build());
firstStatus = getFirstStatus();
@ -169,6 +168,133 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
clearTenantContext();
}
@Test
@SneakyThrows
public void testLastDownloadSetForApprovedFile() {
// Arrange
testData.forwardFileToApprovedState();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Act
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
// Trigger the download processing job
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Assert
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNotNull();
}
@Test
@SneakyThrows
public void testLastDownloadNotSetForNonApprovedFile() {
// Arrange
testData.forwardFile();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Act
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
// Trigger the download processing job
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Assert
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNull();
}
@Test
@SneakyThrows
public void testLastDownloadResetWhenStatusChangedFromApprovedToDifferent() {
// Arrange
testData.forwardFileToApprovedState();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Prepare and process download to set 'last_download'
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Verify that 'last_download' is set
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNotNull();
// Change status from approved to reviewed
fileClient.setStatusUnderReview(testData.getDossierId(), testData.getFileId(), null);
// Assert that 'last_download' is reset
FileStatus updatedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(updatedFileStatus.getLastDownload()).isNull();
}
@Test
@SneakyThrows
public void testLastDownloadRemainsWhenStatusChangedFromApprovedToApproved() {
// Arrange
testData.forwardFileToApprovedState();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Prepare and process download to set 'last_download'
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Verify that 'last_download' is set
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNotNull();
// Change status from approved to approved
fileClient.setStatusApproved(testData.getDossierId(), testData.getFileId(), true);
// Assert that 'last_download' remains set
FileStatus updatedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(updatedFileStatus.getLastDownload()).isNotNull();
}
private void createDossierInService(DossierWithSingleFile testData, List<ReportTemplate> availableTemplates) {
@ -189,18 +315,16 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
.build());
}
private void uploadMockReportTemplate(DossierWithSingleFile testData) {
var template = new MockMultipartFile("test.docx", "zzz".getBytes());
reportTemplateClient.uploadTemplate(template, testData.getDossierTemplateId(), true, true);
}
@SneakyThrows
private void addStoredFileInformationToStorage(FileStatus file, List<ReportTemplate> availableTemplates, String downloadId) {
var storedFileInformationstorageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
var storedFileInformationStorageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
String reportStorageId = "XYZ";
var sivList = new ArrayList<StoredFileInformation>();
@ -210,11 +334,10 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
siv.setTemplateId(availableTemplates.iterator().next().getTemplateId());
sivList.add(siv);
storageService.storeObject(TenantContext.getTenantId(), storedFileInformationstorageId, new ByteArrayInputStream(new ObjectMapper().writeValueAsBytes(sivList)));
storageService.storeObject(TenantContext.getTenantId(), storedFileInformationStorageId, new ByteArrayInputStream(new ObjectMapper().writeValueAsBytes(sivList)));
storageService.storeObject(TenantContext.getTenantId(), reportStorageId, new ByteArrayInputStream(new byte[]{1, 2, 3, 4}));
}
private DownloadStatus getFirstStatus() {
List<DownloadStatus> finalDownloadStatuses = downloadClient.getDownloadStatus().getDownloadStatus();
@ -232,21 +355,15 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
FileStatus file = fileTesterAndProvider.testAndProvideFile(dossier);
public String getDossierTemplateId() {
return dossierTemplate.getId();
}
public String getDossierId() {
return dossier.getId();
}
public String getFileId() {
return file.getFileId();
}
@ -260,6 +377,12 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
assertThatTestFileIsApproved();
}
public void forwardFile() {
fileTesterAndProvider.markFileAsProcessed(getDossierId(), getFileId());
assertThatTestFileIsNew();
}
private void assertThatTestFileIsApproved() {
@ -267,6 +390,12 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
assertThat(persistedFileStatus.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED);
}
private void assertThatTestFileIsNew() {
var persistedFileStatus = fileClient.getFileStatus(getDossierId(), file.getId());
assertThat(persistedFileStatus.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW);
}
}
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java
View File

@ -154,6 +154,8 @@ public class FileStatus {
private FileErrorInfo fileErrorInfo;
@Schema(description = "Shows which version of each mapping the last analysis has been performed")
private Map<String, Integer> componentMappingVersions;
@Schema(description = "Last time the approved file was downloaded")
private OffsetDateTime lastDownload;
@Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.")
public OffsetDateTime getLastOCRTime() {

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DossierAttributeConfig.java
View File

@ -16,6 +16,8 @@ public class DossierAttributeConfig {
private String id;
private String label;
private boolean editable;
private boolean filterable;
private boolean displayedInDossierList;
private String placeholder;
@Builder.Default
private DossierAttributeType type = DossierAttributeType.TEXT;

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -78,6 +78,7 @@ public class FileModel {
private boolean hasHighlights;
private FileErrorInfo fileErrorInfo;
private Map<String, Integer> componentMappingVersions = new HashMap<>();
private OffsetDateTime lastDownloadDate;
public long getFileSize() {

2
persistence-service-v1/persistence-service-shared-mongo-v1/build.gradle.kts
View File

@ -9,7 +9,7 @@ dependencies {
api(project(":persistence-service-shared-api-v1"))
api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.2")
api("com.google.guava:guava:31.1-jre")
api("com.knecon.fforesight:mongo-database-commons:0.17.0") {
api("com.knecon.fforesight:mongo-database-commons:0.18.0") {
exclude(group = "com.knecon.fforesight", module = "tenant-commons")
exclude(group = "org.liquibase.ext", module = "liquibase-mongodb")
}