Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge remote-tracking branch 'origin/master' into RED-9140

Browse Source
This commit is contained in:
Andrei Isvoran 2024-06-21 12:35:30 +03:00
commit 7a024580a6
20 changed files with 258 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java vendored
View File

@ -46,6 +46,7 @@ public class ReanalysisController implements ReanalysisResource {
public void reanalyzeDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) {
accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId);
reanalysisService.reanalyzeDossier(dossierId, force);
auditPersistenceService.audit(AuditRequest.builder()

5
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java vendored
View File

@ -23,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
@ -47,9 +48,9 @@ public class SupportController implements SupportResource {
@Override
public void reanalyzeFiles(String dossierTemplateId, boolean repeatStructureAnalysis) {
public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
reanalysisService.reanalyzeTemplate(dossierTemplateId, repeatStructureAnalysis);
reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings);
}

46
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java vendored
View File

@ -56,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
@ -203,29 +204,32 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
@Override
@SneakyThrows
@PreAuthorize("hasAuthority('" + WRITE_RULES + "')")
public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) {
public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) {
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name;
String nameToUse = Strings.isNullOrEmpty(name) ? file.getOriginalFilename() : name;
if (Strings.isNullOrEmpty(nameToUse)) {
throw new BadRequestException("The provided file name is not valid!");
}
if (Strings.isNullOrEmpty(delimiter) || delimiter.length() != 1) {
throw new BadRequestException("The provided delimiter is not valid! Only a single character is allowed.");
}
char cleanDelimiter = delimiter.charAt(0);
Path mappingFile = saveToFile(file);
String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename();
try {
String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename();
com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
nameToUse,
fileName,
delimiter,
encoding,
mappingFile.toFile());
ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile());
Files.deleteIfExists(mappingFile);
return componentMappingMapper.toModel(metaData);
} finally {
Files.deleteIfExists(mappingFile);
}
return componentMappingMapper.toModel(metaData);
}
@ -238,15 +242,13 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
Path mappingFile = saveToFile(file);
com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId,
componentMappingId,
encoding,
delimiter,
mappingFile.toFile());
try {
ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, componentMappingId, encoding, delimiter, mappingFile.toFile());
Files.deleteIfExists(mappingFile);
return componentMappingMapper.toModel(resultMetaData);
return componentMappingMapper.toModel(resultMetaData);
} finally {
Files.deleteIfExists(mappingFile);
}
}
@ -278,7 +280,11 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
componentMappingService.delete(dossierTemplateId, componentMappingId);
try {
componentMappingService.delete(dossierTemplateId, componentMappingId);
} catch (Exception ignored) {
}
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
}

25
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java vendored
View File

@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.multipart.MultipartFile;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter;
@ -51,10 +52,24 @@ public interface SupportResource {
@PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
@Operation(summary = "Reanalyze all files in dossier template", description = "None")
@Operation(summary = "Reanalyze all files in dossier template", description = """
## Reanalyze Files Endpoint
Use this endpoint to reanalyze all files in a specified Dossier Template. The reanalysis process can be tailored using various filtering options provided in the request body.
### Parameters
- **DossierTemplateId**: Specifies the Dossier Template whose files need to be reanalyzed.
### Request Body Configuration Options
- **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected for reanalysis.
- **fileIds**: List of file IDs to filter. If empty, all files are selected for reanalysis.
- **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
- **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null.
""")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId,
@RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis);
void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings);
@PostMapping(value = ERROR_REANALYSIS_REST_PATH)
@ -115,7 +130,7 @@ public interface SupportResource {
- **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected.
- **fileIds**: List of file IDs to filter. If empty, all files are selected.
- **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS and NER_ENTITIES files.
- **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files.
- **excludeManualRedactions**: Boolean. If true, excludes MANUAL_REDACTIONS files.
- **excludeAnalysisLogs**: Boolean. If true, excludes ENTITY_LOG and COMPONENT_LOG (with overrides) files.
- **excludeFileAttributes**: Boolean. If true, excludes file attributes.
@ -127,7 +142,7 @@ public interface SupportResource {
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@ResponseBody
@PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. If the request parameter dossierTemplateId is set, the files will be imported into the existing DossierTemplate.")
@Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
void importFiles(@RequestPart(name = "file") MultipartFile file);

2
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java vendored
View File

@ -140,7 +140,7 @@ public interface DossierTemplateResource {
@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name,
@Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding,
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter);
@Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter);
@Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None")

2
persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml
View File

@ -594,7 +594,7 @@ paths:
- Component Definitions
summary: Create new component definitions
description: |
Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case
Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case
that can't be updated after the creation. The rank is used to determine the order in which the components are displayed. The component's rank will
automatically be appended at the end based on the current number of components of this dossier template.
parameters:

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java
View File

@ -24,6 +24,7 @@ public class FileExchangeNames {
public static Definition PAGES = new Definition(FileType.DOCUMENT_PAGES);
public static Definition TEXT = new Definition(FileType.DOCUMENT_TEXT);
public static Definition POSITIONS = new Definition(FileType.DOCUMENT_POSITION);
public static Definition SIMPLIFIED_TEXT = new Definition(FileType.SIMPLIFIED_TEXT);
public static Definition NER_ENTITIES = new Definition(FileType.NER_ENTITIES);
public static Definition TABLES = new Definition(FileType.TABLES);

29
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java
View File

@ -3,18 +3,24 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
@ -56,12 +62,18 @@ public class FileExchangeExportService {
ObjectMapper mapper;
RabbitTemplate rabbitTemplate;
EntityTypeExportService entityTypeExportService;
DossierIdFileIdRequestValidator requestValidator;
@Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
public DownloadResponse prepareExportDownload(String dossierTemplateId, FileExchangeExportRequest request) {
var mimeType = "application/zip";
String dossierTemplateName = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getName();
DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId);
String dossierTemplateName = dossierTemplate.getName();
requestValidator.validateRequestOrThrow404(dossierTemplateId, request.dossierIds(), request.fileIds());
String downloadFilename = dossierTemplateName + "_file-exchange.zip";
String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_file-exchange");
@ -118,14 +130,25 @@ public class FileExchangeExportService {
@Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
if (!request.dossierIds().contains(dossier.getId()) //
&& files.stream()
.noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
// dossier has no files in requested files and dossier not explicitly requested -> don't export it.
return;
}
Path dossierFolder = folder.resolve(dossier.getId());
archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
for (FileModel fileEntity : fileStatusManagementService.getDossierStatus(dossier.getId())) {
for (FileModel fileEntity : files) {
if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
continue;
}
addFileToArchive(archiver, dossierFolder, request, fileEntity);
}
List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);
String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString();
@ -164,6 +187,7 @@ public class FileExchangeExportService {
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES);
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT);
}
addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE);
@ -220,6 +244,7 @@ public class FileExchangeExportService {
archiver.addEntry(archiveModel);
}
@Observed(name = "FileExchangeExportService", contextualName = "store-archive")
private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) {

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java
View File

@ -97,6 +97,7 @@ public class FileExchangeArchiveReader {
FileExchangeNames.PAGES,
FileExchangeNames.TEXT,
FileExchangeNames.POSITIONS,
FileExchangeNames.SIMPLIFIED_TEXT,
FileExchangeNames.NER_ENTITIES,
FileExchangeNames.TABLES,
FileExchangeNames.IMAGES,

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java
View File

@ -124,7 +124,7 @@ public class ComponentMappingService {
}
private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws CsvException, BadRequestException, IOException {
private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException {
Path tempFile = Files.createTempFile("mapping", ".tmp");
@ -140,6 +140,10 @@ public class ComponentMappingService {
columnLabels = rows.remove(0); // remove header row
if (Arrays.stream(columnLabels).distinct().count() < columnLabels.length) {
throw new BadRequestException("Column labels may not contain duplicates!");
}
numberOfLines = (int) reader.getLinesRead() - 1; // subtract header row
rows.sort(CSV_SORTER);
@ -150,10 +154,12 @@ public class ComponentMappingService {
} catch (IOException e) {
throw new BadRequestException("Error while sorting the csv file", e);
} catch (CsvException e) {
throw new BadRequestException("The file could not be parsed as a csv file", e);
} finally {
Files.deleteIfExists(tempFile);
}
Files.deleteIfExists(tempFile);
return new CsvStats(Arrays.asList(columnLabels), numberOfLines);
}

59
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java Normal file
View File

@ -0,0 +1,59 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DossierIdFileIdRequestValidator {
DossierManagementService dossierManagementService;
FileStatusManagementService fileStatusManagementService;
@Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
if (!dossierIds.isEmpty()) {
Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
if (!nonAvailableDossiers.isEmpty()) {
throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId));
}
}
if (!fileIds.isEmpty()) {
Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId)
.stream()
.filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
.map(FileModel::getId)
.collect(Collectors.toSet());
Set<String> nonAvailableFiles = Sets.difference(fileIds, availableFileIds);
if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) {
throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplateId));
}
if (!nonAvailableFiles.isEmpty()) {
throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s",
String.join(", ", nonAvailableFiles),
String.join(", ", dossierIds),
dossierTemplateId));
}
}
}
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java
View File

@ -119,6 +119,13 @@ public class DossierManagementService {
}
@Transactional
public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
}
public DossierInformation getDossierInformation(List<String> filteredDossierIds) {

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java
View File

@ -1,5 +1,11 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -17,12 +23,6 @@ import jakarta.validation.ConstraintViolationException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
/**
* Provides the internal interface between dossier request and the actual persistence.
*/
@ -143,6 +143,11 @@ public class DossierService {
return dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
}
public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId);
}
public Set<DossierChange> changesSince(OffsetDateTime since) {

40
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java
View File

@ -9,6 +9,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -40,6 +41,7 @@ public class ReanalysisService {
private final IndexingService indexingService;
private final PDFTronClient pDFTronRedactionClient;
private final FileManagementStorageService fileManagementStorageService;
private final DossierIdFileIdRequestValidator requestValidator;
private final Predicate<FileModel> validFilesFilter = fileStatus -> !fileStatus.isSoftOrHardDeleted() && !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED);
private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
@ -54,7 +56,7 @@ public class ReanalysisService {
public void reanalyzeDossier(String dossierId, boolean force) {
var relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter);
List<FileModel> relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter);
reanalyseFiles(force, relevantFiles);
}
@ -265,13 +267,41 @@ public class ReanalysisService {
}
public void reanalyzeTemplate(String dossierTemplateId, boolean repeatStructureAnalysis) {
public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
fileStatusService.getDossierTemplateStatus(dossierTemplateId)
requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds());
var files = fileStatusService.getDossierTemplateStatus(dossierTemplateId)
.stream()
.filter(file -> !file.isSoftOrHardDeleted())
.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, repeatStructureAnalysis));
.filter(file -> isInList(file, reanalysisSettings))
.filter(reanalysisSettings.fileStatusFilter().asPredicate())
.peek(file -> log.info("Reanalyzing file {}", file.getId()))
.toList();
validateFilesForReanalysis(files);
files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis()));
}
private void validateFilesForReanalysis(List<FileModel> files) {
for (var file : files) {
if (file.isSoftOrHardDeleted()) {
throw new BadRequestException("Cannot reanalyse deleted file!");
}
if (file.getWorkflowStatus() == WorkflowStatus.APPROVED) {
throw new BadRequestException("Cannot reanalyse approved file!");
}
}
}
private boolean isInList(FileModel file, ReanalysisSettings reAnalysisSettings) {
return (reAnalysisSettings.fileIds().isEmpty() || reAnalysisSettings.fileIds().contains(file.getId())) //
&& (reAnalysisSettings.dossierIds().isEmpty() || reAnalysisSettings.dossierIds().contains(file.getDossierId()));
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java
View File

@ -195,6 +195,12 @@ public class DossierPersistenceService {
}
public Set<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId);
}
@Transactional
public void hardDelete(String dossierId) {

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
@ -84,6 +85,10 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
List<DossierEntity> findByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
@Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId")
Set<String> findIdsByDossierTemplateId(String dossierTemplateId);
@Modifying
@Query("update DossierEntity d set d.watermarkId = null where d.watermarkId = :watermarkId")
int countDeleteWatermark(@Param("watermarkId") long watermarkId);

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java
View File

@ -123,7 +123,7 @@ public class FileExchangeImportExportTest extends AbstractPersistenceServerServi
private FileExchangeExportRequest buildDefaultRequest() {
return new FileExchangeExportRequest(Collections.emptyList(), Collections.emptyList(), false, false, false, false, true);
return new FileExchangeExportRequest(Collections.emptySet(), Collections.emptySet(), false, false, false, false, true);
}

7
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java
View File

@ -1,13 +1,14 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model;
import java.util.List;
import java.util.Set;
import io.swagger.v3.oas.annotations.media.Schema;
public record FileExchangeExportRequest(
@Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") List<String> dossierIds,
@Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") List<String> fileIds,
@Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS and NER_ENTITIES file will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles,
@Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") Set<String> dossierIds,
@Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") Set<String> fileIds,
@Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles,
@Schema(description = "If set to true, the MANUAL_REDACTIONS file will be excluded from the export.", defaultValue = "false") boolean excludeManualRedactions,
@Schema(description = "If set to true, the ENTITY_LOG and COMPONENT_LOG (with its overrides) will be excluded from the export.", defaultValue = "false") boolean excludeAnalysisLogs,
@Schema(description = "If set to true, the File Attributes will be excluded from the export.", defaultValue = "false") boolean excludeFileAttributes,

19
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java
View File

@ -2,7 +2,9 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Predicate;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -29,4 +31,21 @@ public class FileStatusFilter {
this.includeHardDeletedFiles = false;
}
public Predicate<FileModel> asPredicate() {
if (this.getProcessingStatusList() == null) {
this.setProcessingStatusList(new ArrayList<>());
}
if (this.getWorkflowStatusList() == null) {
this.setWorkflowStatusList(new ArrayList<>());
}
return fileStatus -> (this.getProcessingStatusList().isEmpty() || this.getProcessingStatusList().contains(fileStatus.getProcessingStatus()))
&& (this.getWorkflowStatusList().isEmpty() || this.getWorkflowStatusList().contains(fileStatus.getWorkflowStatus()))
&& (this.isIncludeSoftDeletedFiles() || fileStatus.getDeleted() == null)
&& (this.isIncludeHardDeletedFiles() || fileStatus.getHardDeletedTime() == null);
}
}

21
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java Normal file
View File

@ -0,0 +1,21 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model;
import java.util.Optional;
import java.util.Set;
import io.swagger.v3.oas.annotations.media.Schema;
public record ReanalysisSettings(
@Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set<String> dossierIds,
@Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set<String> fileIds,
@Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis,
@Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter
) {
public FileStatusFilter fileStatusFilter() {
return Optional.ofNullable(fileStatusFilter)
.orElse(new FileStatusFilter());
}
}