Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'master' into feature/RED-10071

Browse Source 
# Conflicts:
#	persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
This commit is contained in:
corinaolariu 2024-10-15 10:34:12 +03:00
commit a65e9e1086
33 changed files with 1052 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java vendored
View File

@ -266,11 +266,14 @@ public class FileManagementController implements FileManagementResource {
FileModel fileModel = fileStatusManagementService.getFileStatus(fileId);
if (!fileModel.isExcludedFromAutomaticAnalysis()) {
if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) {
reanalysisService.ocrFile(dossierId, fileId, true);
} else {
// Re- ocr is files with pdftron ocr service is breaking files and not needed with azure.
// TODO find a better solution for this.
// if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) {
// reanalysisService.ocrFile(dossierId, fileId, true);
// } else {
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
}
// }
}
auditPersistenceService.audit(AuditRequest.builder()

42
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java vendored
View File

@ -46,6 +46,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
@ -211,17 +212,40 @@ public class ManualRedactionController implements ManualRedactionResource {
var dossier = dossierManagementService.getDossierById(dossierId, false, false);
dictionaryPersistenceService.getType(TypeIdUtils.toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId()));
fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest);
if (!addRedactionRequest.isRectangle()) {
fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest);
EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest);
return ManualRedactionResponse.builder()
.manualAnnotationResponses(List.of(ManualAnnotationResponse.builder()
.annotationId(manualRedactionService.getNewAnnotationId())
.entityLogEntry(entityLogEntry)
.build()))
.build();
EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest);
return ManualRedactionResponse.builder()
.manualAnnotationResponses(List.of(ManualAnnotationResponse.builder()
.annotationId(manualRedactionService.getNewAnnotationId())
.entityLogEntry(entityLogEntry)
.build()))
.build();
} else {
Set<AddRedactionRequestModel> addRedactionRequestModels = addRedactionRequest.getPageNumbers()
.stream()
.map(page -> AddRedactionRequestModel.builder()
.type(addRedactionRequest.getType())
.value(addRedactionRequest.getValue())
.reason(addRedactionRequest.getReason())
.legalBasis(addRedactionRequest.getLegalBasis())
.positions(addRedactionRequest.getPositions()
.stream()
.map(rectangle -> Rectangle.createRectangle(rectangle.getTopLeftX(),
rectangle.getTopLeftY(),
rectangle.getWidth(),
rectangle.getHeight(),
page))
.toList())
.comment(addRedactionRequest.getComment() != null ? AddCommentRequestModel.builder().text(addRedactionRequest.getComment()).build() : null)
.section(addRedactionRequest.getSection())
.rectangle(true)
.build())
.collect(Collectors.toSet());
return addRedactionBulk(dossierId, fileId, addRedactionRequestModels);
}
}

22
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java vendored
View File

@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
@ -48,6 +49,7 @@ public class SupportController implements SupportResource {
private final FileStatusManagementService fileStatusManagementService;
private final FileExchangeExportService fileExchangeExportService;
private final FileExchangeImportService fileExchangeImportService;
private final DatasetExchangeService datasetExchangeService;
@Override
@ -127,6 +129,13 @@ public class SupportController implements SupportResource {
}
@Override
public DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) {
return datasetExchangeService.prepareExport(dossierTemplateId);
}
@Override
public DownloadResponse exportFiles(String dossierTemplateId, FileExchangeExportRequest exportRequest) {
@ -147,4 +156,17 @@ public class SupportController implements SupportResource {
return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes);
}
@Override
@PreAuthorize("hasAuthority('" + IMPORT_FILES + "')")
public ImportResponse importDataset(MultipartFile file) {
byte[] bytes;
try {
bytes = file.getBytes();
} catch (IOException e) {
throw new BadRequestException("File could not be read and is likely corrupted.", e);
}
return datasetExchangeService.importDataset(KeycloakSecurity.getUserId(), bytes);
}
}

32
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java vendored
View File

@ -9,6 +9,7 @@ import java.util.ArrayList;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
@ -42,15 +43,17 @@ import lombok.experimental.FieldDefaults;
@RestController
@RequiredArgsConstructor
@Tag(name = "4. Component endpoints", description = "Provides operations related to components")
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ComponentControllerV2 implements ComponentResource {
ComponentLogService componentLogService;
UserService userService;
StatusController statusController;
FileStatusService fileStatusService;
DossierTemplatePersistenceService dossierTemplatePersistenceService;
ComponentMapper componentMapper = ComponentMapper.INSTANCE;
private final ComponentLogService componentLogService;
private final UserService userService;
private final StatusController statusController;
private final FileStatusService fileStatusService;
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final ComponentMapper componentMapper = ComponentMapper.INSTANCE;
@Value("${application.type}")
private String applicationType;
@Override
@ -59,8 +62,9 @@ public class ComponentControllerV2 implements ComponentResource {
@PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) {
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
checkApplicationType();
validateUserRoles(KeycloakSecurity.getUserId());
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
var componentLog = componentLogService.getComponentLog(dossierId, fileId);
return componentMapper.toFileComponents(componentLog, dossierTemplateId, dossierId, fileId, fileStatusService.getFileName(fileId), includeDetails);
@ -85,6 +89,7 @@ public class ComponentControllerV2 implements ComponentResource {
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) {
checkApplicationType();
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
var dossierFiles = statusController.getDossierStatus(dossierId);
return new FileComponentsList(dossierFiles.stream()
@ -100,6 +105,7 @@ public class ComponentControllerV2 implements ComponentResource {
@PathVariable(FILE_ID_PARAM) String fileId,
@RequestBody Component override) {
checkApplicationType();
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
componentLogService.overrideComponent(dossierId, fileId, componentMapper.toComponentLogEntry(override));
@ -112,6 +118,7 @@ public class ComponentControllerV2 implements ComponentResource {
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@PathVariable(FILE_ID_PARAM) String fileId) {
checkApplicationType();
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
FileModel status = fileStatusService.getStatus(fileId);
@ -138,9 +145,18 @@ public class ComponentControllerV2 implements ComponentResource {
@PathVariable(FILE_ID_PARAM) String fileId,
@RequestBody RevertOverrideRequest revertOverrideRequest) {
checkApplicationType();
dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId);
componentLogService.revertOverrides(dossierId, fileId, revertOverrideRequest);
}
private void checkApplicationType() {
if(!applicationType.equals("DocuMine")) {
throw new NotAllowedException("Components can only be accessed in DocuMine");
}
}
}

24
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java vendored
View File

@ -1,6 +1,7 @@
package com.iqser.red.persistence.service.v2.external.api.impl.controller;
import java.util.List;
import java.util.Optional;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
@ -8,7 +9,11 @@ import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.DownloadController;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RemoveDownloadRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.DownloadStatus;
import com.iqser.red.service.persistence.service.v2.api.external.model.DownloadStatusList;
@ -26,10 +31,13 @@ public class DownloadControllerV2 implements DownloadResource {
private final DownloadController downloadController;
private final DownloadStatusPersistenceService downloadStatusPersistenceService;
private final UserService userService;
@Transactional
public DownloadStatusList getDownloadStatusList() {
validateUserRoles(KeycloakSecurity.getUserId());
var downloads = downloadStatusPersistenceService.getStatusesByUser(KeycloakSecurity.getUserId());
return new DownloadStatusList(downloads.stream().map(
@ -61,6 +69,7 @@ public class DownloadControllerV2 implements DownloadResource {
@Transactional
public DownloadStatus getDownloadStatus(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) {
validateUserRoles(KeycloakSecurity.getUserId());
var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId);
return DownloadStatus.builder()
@ -88,6 +97,7 @@ public class DownloadControllerV2 implements DownloadResource {
public void deleteDownload(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) {
validateUserRoles(KeycloakSecurity.getUserId());
var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId);
downloadController.deleteDownloadStatus(new RemoveDownloadRequest(List.of(status.getStorageId())));
}
@ -95,8 +105,22 @@ public class DownloadControllerV2 implements DownloadResource {
public void download(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) {
validateUserRoles(KeycloakSecurity.getUserId());
var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId);
downloadController.downloadFile(status.getStorageId());
}
private void validateUserRoles(String userId) {
Optional<User> userOptional = userService.getUserById(userId);
if (userOptional.isPresent()) {
if (userOptional.get().getRoles()
.stream()
.noneMatch(ApplicationRoles.VALID_MEMBER_ROLES::contains)) {
throw new NotAllowedException("User doesn't have appropriate roles");
}
}
}
}

23
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java vendored
View File

@ -34,6 +34,7 @@ public interface SupportResource {
String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status/filter";
String FILE_EXCHANGE_REST_PATH = ExternalApi.BASE_PATH + "/file-exchange";
String DATASET_EXCHANGE = ExternalApi.BASE_PATH + "/dataset-exchange";
String BULK_REST_PATH = "/bulk";
@ -143,6 +144,21 @@ public interface SupportResource {
DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest);
@ResponseStatus(value = HttpStatus.OK)
@ResponseBody
@PostMapping(value = DATASET_EXCHANGE
+ EXPORT
+ DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """
## Export Preview Files Endpoint
Use this endpoint to export all files of a DossierTemplate as a Preview file containing all applied annotations as redaction annotations with additional data
The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive.
""")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@ -150,4 +166,11 @@ public interface SupportResource {
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")})
ImportResponse importDataset(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file);
}

6
persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml
View File

@ -3629,7 +3629,7 @@ components:
modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
validFrom: 2020-01-01T00:00:00.000+00:00
validTo: 2030-12-31T23:59:59.999+00:00
dossierTemplateStatus: ACTIVE
status: ACTIVE
removeWatermark: false
keepImageMetadata: true
ocrByDefault: true
@ -3750,7 +3750,7 @@ components:
modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
validFrom: 2020-01-01T00:00:00.000+00:00
validTo: 2030-12-31T23:59:59.999+00:00
dossierTemplateStatus: ACTIVE
status: ACTIVE
removeWatermark: false
keepImageMetadata: true
ocrByDefault: true
@ -3768,7 +3768,7 @@ components:
modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d
validFrom: 2023-01-01T00:00:00.000+00:00
validTo: 2033-12-31T23:59:59.999+00:00
dossierTemplateStatus: ACTIVE
status: ACTIVE
removeWatermark: true
keepImageMetadata: true
ocrByDefault: true

6
persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml
View File

@ -2076,7 +2076,7 @@ components:
modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
validFrom: 2020-01-01T00:00:00.000+00:00
validTo: 2030-12-31T23:59:59.999+00:00
dossierTemplateStatus: ACTIVE
status: ACTIVE
removeWatermark: false
keepImageMetadata: false
ocrByDefault: false
@ -2212,7 +2212,7 @@ components:
modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6
validFrom: 2020-01-01T00:00:00.000+00:00
validTo: 2030-12-31T23:59:59.999+00:00
dossierTemplateStatus: ACTIVE
status: ACTIVE
removeWatermark: false
keepImageMetadata: false
ocrByDefault: false
@ -2231,7 +2231,7 @@ components:
modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d
validFrom: 2023-01-01T00:00:00.000+00:00
validTo: 2033-12-31T23:59:59.999+00:00
dossierTemplateStatus: ACTIVE
status: ACTIVE
removeWatermark: true
keepImageMetadata: false
ocrByDefault: true

57
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java Normal file
View File

@ -0,0 +1,57 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models;
import java.util.LinkedList;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DatasetExchangeImportModel {
private ImportTemplateResult importTemplateResult = ImportTemplateResult.builder().build();
private List<Dossier> dossiers = new LinkedList<>();
@Builder
@Getter
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public static class Dossier {
String name;
@Builder.Default
List<File> files = new LinkedList<>();
}
@Getter
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public static class File {
String name;
byte[] data;
}
public DatasetExchangeImportModel.Dossier getOrCreateDossier(String name) {
for (Dossier dossier : dossiers) {
if (dossier.getName().equals(name)) {
return dossier;
}
}
Dossier dossier = Dossier.builder().name(name).build();
dossiers.add(dossier);
return dossier;
}
}

183
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java Normal file
View File

@ -0,0 +1,183 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeArchivalService.SIZE_THRESHOLD;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DatasetExchangeArchiveReader;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.UploadService;
import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DatasetExchangeService {
DossierTemplateManagementService dossierTemplateManagementService;
DownloadStatusPersistenceService downloadStatusPersistenceService;
FileStatusPersistenceService fileStatusPersistenceService;
DossierPersistenceService dossierPersistenceService;
ColorsService colorsService;
RabbitTemplate rabbitTemplate;
FileManagementServiceSettings settings;
DossierTemplateImportService dossierTemplateImportService;
DossierCreatorService dossierCreatorService;
ReportTemplatePersistenceService reportTemplateService;
UploadService uploadService;
DownloadPreparationService downloadPreparationService;
FileManagementStorageService storageService;
public DownloadResponse prepareExport(String dossierTemplateId) {
var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId);
String downloadFilename = template.getName() + "_PREVIEW_FILES.zip";
String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES");
List<FileEntity> files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId);
List<DossierEntity> dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId);
if (dossiers.isEmpty()) {
throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId);
}
ColorsEntity colors = colorsService.getColors(dossierTemplateId);
downloadPreparationService.clearRedactionStatusEntries(storageId);
if (storageService.objectExists(storageId)) {
storageService.deleteObject(storageId);
}
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.createStatus(KeycloakSecurity.getUserId(),
storageId,
dossiers.get(0),
downloadFilename,
"application/zip",
files.stream()
.map(FileEntity::getId)
.toList(),
Set.of(DownloadFileType.DATASET_EXPORT),
Collections.emptyList(),
colors.getPreviewColor());
RedactionMessage.RedactionMessageBuilder builder = RedactionMessage.builder()
.appliedRedactionColor(colors.getRedactionColor())
.redactionPreviewColor(colors.getPreviewColor())
.keepHiddenText(true)
.keepOverlappingObjects(true)
.keepImageMetaData(true)
.downloadId(storageId)
.redactionTypes(List.of(RedactionType.PREVIEW));
downloadStatus.getFiles()
.forEach(fileEntity -> {
RedactionMessage message = builder.dossierId(fileEntity.getDossierId())
.fileId(fileEntity.getId())
.unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED)
.build();
log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", storageId, fileEntity.getId());
rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_REQUEST_EXCHANGE, TenantContext.getTenantId(), message);
});
downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.GENERATING);
return new DownloadResponse(storageId);
}
@SneakyThrows
public ImportResponse importDataset(String userId, byte[] archive) {
DatasetExchangeArchiveReader datasetExchangeArchiveReader = new DatasetExchangeArchiveReader(userId);
try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) {
zipEntryIterator.forEachRemaining(datasetExchangeArchiveReader::handleZipEntryData);
}
DatasetExchangeImportModel importModel = datasetExchangeArchiveReader.getDatasetExchangeImportModel();
TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importModel.getImportTemplateResult());
for (DatasetExchangeImportModel.Dossier dossier : importModel.getDossiers()) {
String dossierId = getDossierForImport(templateImportInfo, dossier, userId);
for (DatasetExchangeImportModel.File file : dossier.getFiles()) {
uploadService.processSingleFile(dossierId, file.getName(), file.getData(), false, false);
}
}
return new ImportResponse(templateImportInfo.getDossierTemplateId());
}
private String getDossierForImport(TemplateImportInfo templateImportInfo, DatasetExchangeImportModel.Dossier dossier, String userId) {
String dossierTemplateId = templateImportInfo.getDossierTemplateId();
CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder()
.dossierName(dossier.getName())
.description(dossier.getName())
.downloadFileTypes(Collections.emptySet())
.dossierTemplateId(dossierTemplateId)
.requestingUser(userId)
.reportTemplateIds(reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId())
.stream()
.map(ReportTemplateEntity::getTemplateId)
.toList())
.build();
Dossier importedDossier = null;
int retries = 0;
while (importedDossier == null && retries < 100) {
try {
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
} catch (ConflictException e) {
retries++;
request.setDossierName(String.format("%s (%d)", dossier.getName(), retries));
}
}
if (importedDossier == null) {
throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries));
}
return importedDossier.getId();
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java
View File

@ -67,7 +67,7 @@ public class DossierImportService {
Dossier importedDossier = null;
int retries = 0;
while (importedDossier == null && retries < 10) {
while (importedDossier == null && retries < 100) {
try {
importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId);
} catch (ConflictException e) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java
View File

@ -72,6 +72,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@ -150,8 +151,9 @@ public class DossierTemplateExportService {
}
@SneakyThrows
@Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive")
public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException {
public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) {
// add dossier template name and meta data
fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder,

50
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java Normal file
View File

@ -0,0 +1,50 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel;
import lombok.AccessLevel;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DatasetExchangeArchiveReader {
DossierTemplateArchiveReader dossierTemplateArchiveReader;
DatasetExchangeImportModel datasetExchangeImportModel;
public DatasetExchangeArchiveReader(String userId) {
dossierTemplateArchiveReader = new DossierTemplateArchiveReader(userId, null, false);
datasetExchangeImportModel = new DatasetExchangeImportModel();
}
public DatasetExchangeImportModel getDatasetExchangeImportModel() {
datasetExchangeImportModel.setImportTemplateResult(dossierTemplateArchiveReader.buildResult());
return datasetExchangeImportModel;
}
@SneakyThrows
public void handleZipEntryData(ZipEntryData zipEntryData) {
if (zipEntryData.getBaseFolder().equals(FileExchangeNames.TEMPLATE_FOLDER)) {
dossierTemplateArchiveReader.handleZipEntryData(zipEntryData);
} else if (zipEntryData.getDepth() >= 1) { // in dossier folders
String dossierName = zipEntryData.getBaseFolder().replaceAll("\\s*\\([1-9]+\\)", "");
String fileName = zipEntryData.getFileName();
if (!fileName.endsWith(".pdf")) {
return;
}
var dossier = datasetExchangeImportModel.getOrCreateDossier(dossierName);
dossier.getFiles().add(new DatasetExchangeImportModel.File(fileName, zipEntryData.data()));
}
}
}

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java
View File

@ -1,6 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders;
import java.io.*;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.UUID;
@ -70,7 +74,7 @@ public class ZipEntryIterator implements Iterator<ZipEntryData>, AutoCloseable {
double compressionRatio = (double) totalSizeEntry / nextEntry.getCompressedSize();
if (compressionRatio > compressionThresholdRatio) {
throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio );
throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio);
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java
View File

@ -53,7 +53,9 @@ public final class ApplicationRoles {
DOWNLOAD_REDACTION_PREVIEW_FILE,
DOWNLOAD_REPORT_TEMPLATE,
EXCLUDE_INCLUDE_FILE,
EXCLUDE_INCLUDE_PAGES, GET_REPORT_TEMPLATES, GET_SIMILAR_IMAGES,
EXCLUDE_INCLUDE_PAGES,
GET_REPORT_TEMPLATES,
GET_SIMILAR_IMAGES,
MANAGE_USER_PREFERENCES,
MANAGE_VIEWED_PAGES,
PROCESS_DOWNLOAD,
@ -105,7 +107,9 @@ public final class ApplicationRoles {
DELETE_DICTIONARY_TYPE,
DELETE_REPORT_TEMPLATE,
DOWNLOAD_REPORT_TEMPLATE,
GET_REPORT_TEMPLATES, MANAGE_USER_PREFERENCES, GET_SIMILAR_IMAGES,
GET_REPORT_TEMPLATES,
MANAGE_USER_PREFERENCES,
GET_SIMILAR_IMAGES,
READ_COLORS,
READ_DICTIONARY_TYPES,
READ_DIGITAL_SIGNATURE,
@ -147,7 +151,9 @@ public final class ApplicationRoles {
public static final Set<String> RED_USER_ADMIN_ACTION_ROLES = Sets.newHashSet(MANAGE_USER_PREFERENCES,
READ_ALL_USERS,
READ_APP_CONFIG,
READ_GENERAL_CONFIGURATION, READ_GENERAL_CONFIGURATION, GET_SIMILAR_IMAGES,
READ_GENERAL_CONFIGURATION,
READ_GENERAL_CONFIGURATION,
GET_SIMILAR_IMAGES,
READ_NOTIFICATIONS,
READ_USERS,
UPDATE_MY_PROFILE,

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -73,6 +73,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddR
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.llm.service.LlmNerMessage;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
@ -1060,6 +1061,7 @@ public class FileStatusService {
.section(addRedactionBulkLocalRequestModel.getSection())
.pageNumbers(addRedactionBulkLocalRequestModel.getPageNumbers())
.comment(addRedactionBulkLocalRequestModel.getComment())
.userId(KeycloakSecurity.getUserId())
.build();
addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest);
}

26
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service.download;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@ -21,6 +22,8 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetai
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -29,12 +32,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.download
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadRedactionFileStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
@ -68,6 +72,8 @@ public class DownloadPreparationService {
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository;
private final WebsocketService websocketService;
private final DossierPersistenceService dossierPersistenceService;
private final DossierTemplateExportService dossierTemplateExportService;
@Value("${storage.backend}")
private String storageBackend;
@ -134,7 +140,7 @@ public class DownloadPreparationService {
for (var downloadFileType : downloadFileTypes) {
switch (downloadFileType) {
case REDACTED -> result.add(RedactionType.REDACTED);
case PREVIEW -> result.add(RedactionType.PREVIEW);
case PREVIEW, DATASET_EXPORT -> result.add(RedactionType.PREVIEW);
case OPTIMIZED_PREVIEW -> result.add(RedactionType.OPTIMIZED_PREVIEW);
case DELTA_PREVIEW -> result.add(RedactionType.DELTA);
default -> {
@ -198,7 +204,7 @@ public class DownloadPreparationService {
var redactionFileResults = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(downloadId);
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadId);
var storedFileInformations = getStoredFileInformation(downloadId);
List<StoredFileInformation> storedFileInformations = getStoredFileInformation(downloadId);
try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
@ -276,6 +282,11 @@ public class DownloadPreparationService {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(file.getFilename(), "redacted"), //
getRedacted(file.getId(), redactionFileResult.get().getDetails())));
}
if (downloadFileType.equals(DownloadFileType.DATASET_EXPORT)) {
String dossierName = dossierPersistenceService.getDossierNameById(file.getDossierId());
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierName, file.getFilename(),//
getPreview(file.getId(), redactionFileResult.get().getDetails())));
}
}
log.info("Successfully added file {}/{} for downloadId {}, took {}",
@ -285,6 +296,10 @@ public class DownloadPreparationService {
System.currentTimeMillis() - start);
i++;
}
if (downloadStatus.getDownloadFileTypes().contains(DownloadFileType.DATASET_EXPORT)) {
DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadStatus.getDossier().getDossierTemplateId());
dossierTemplateExportService.addDossierTemplateToArchive(fileSystemBackedArchiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
}
log.info("Successfully added {} files for downloadId {}, took {}",
downloadStatus.getFiles()
.stream()
@ -381,7 +396,10 @@ public class DownloadPreparationService {
} else {
storageId = generateReportJsonStorageIdForAzure(downloadId);
}
if (!fileManagementStorageService.objectExists(storageId)) {
log.warn("Could not find stored file information with id {}", storageId);
return new ArrayList<>();
}
return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() {
});
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java
View File

@ -22,8 +22,10 @@ public class DownloadReportCleanupService {
public void deleteTmpReportFiles(Collection<String> storageIds) {
for (String storageId : storageIds) {
fileManagementStorageService.deleteObject(storageId);
log.info("Deleted tmp report file {}", storageId);
if (fileManagementStorageService.objectExists(storageId)) {
fileManagementStorageService.deleteObject(storageId);
log.info("Deleted tmp report file {}", storageId);
}
}
}

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java
View File

@ -209,7 +209,7 @@ public class DossierPersistenceService {
@Transactional
public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) {
public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) {
dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
@ -275,4 +275,11 @@ public class DossierPersistenceService {
return dossierRepository.findDossierTemplateId(dossierId);
}
public String getDossierNameById(String dossierId) {
return dossierRepository.findDossierNameById(dossierId)
.orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -38,7 +38,7 @@ public class DownloadStatusPersistenceService {
}
public void createStatus(String userId,
public DownloadStatusEntity createStatus(String userId,
String storageId,
DossierEntity dossier,
String filename,
@ -62,7 +62,7 @@ public class DownloadStatusPersistenceService {
downloadStatus.setRedactionPreviewColor(redactionPreviewColor);
downloadStatus.setStatus(DownloadStatusValue.QUEUED);
downloadStatusRepository.save(downloadStatus);
return downloadStatusRepository.save(downloadStatus);
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.springframework.data.jpa.repository.JpaRepository;
@ -110,4 +111,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
@Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold")
List<DossierEntity> findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
@Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java
View File

@ -10,6 +10,11 @@ import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContext;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.context.SecurityContextImpl;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
@ -73,6 +78,11 @@ public class SearchTermOccurrencesResponseReceiver {
public void receive(BulkLocalResponse response) {
Authentication authentication = new UsernamePasswordAuthenticationToken(response.getUserId(), null, null);
SecurityContextImpl securityContext = new SecurityContextImpl();
securityContext.setAuthentication(authentication);
SecurityContextHolder.setContext(securityContext);
var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false);
Set<AddRedactionRequestModel> addRedactionRequests = response.getFoundTerms()

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java
View File

@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
@ -50,7 +51,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
this.rethrowExceptions = rethrowExceptions;
tempFile = FileUtils.createTempFile("archive", ".zip");
zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile));
zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile), StandardCharsets.UTF_8);
}

4
persistence-service-v1/persistence-service-server-v1/build.gradle.kts
View File

@ -41,19 +41,19 @@ description = "persistence-service-server-v1"
tasks.named<BootBuildImage>("bootBuildImage") {
builder.set("docker-proxy.knecon.com/paketobuildpacks/builder:base")
runImage.set("docker-proxy.knecon.com/paketobuildpacks/run:base-cnb")
environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // FileInputStream or ZipInputStream does not seem to care for file.encoding
imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
if (project.hasProperty("buildbootDockerHostNetwork")) {
network.set("host")
}
docker {
docker {
if (project.hasProperty("buildbootDockerHostNetwork")) {
bindHostToBuilder.set(true)
}

161
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentControllerV2Test.java
View File

@ -1,18 +1,21 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.*;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.MockitoAnnotations;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import org.springframework.test.util.ReflectionTestUtils;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController;
import com.iqser.red.persistence.service.v2.external.api.impl.controller.ComponentControllerV2;
@ -22,17 +25,26 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import lombok.SneakyThrows;
public class ComponentControllerV2Test {
public static final String USER_ID = "user-111";
public static final String FILE_ID = "file-789";
public static final String DOSSIER_ID = "dossier-456";
public static final String DOSSIER_TEMPLATE_ID = "template-123";
@InjectMocks
private ComponentControllerV2 componentControllerV2;
@Mock
private UserService userService;
@Mock
@ -43,17 +55,22 @@ public class ComponentControllerV2Test {
private DossierTemplatePersistenceService dossierTemplatePersistenceService;
@Mock
private StatusController statusController;
@Mock
private ComponentLog mockComponentLog;
@BeforeEach
@SneakyThrows
void setUp() {
MockitoAnnotations.openMocks(this);
openMocks(this);
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "DocuMine");
}
@Test
void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles() {
void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_getComponents() {
User invalidUser = new User();
invalidUser.setUserId(USER_ID);
@ -64,43 +81,129 @@ public class ComponentControllerV2Test {
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertThrows(NotAllowedException.class, () -> {
componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false);
});
assertThrows(NotAllowedException.class, () -> componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false));
}
}
@Test
void shouldNotThrowNotAllowedExceptionIfUserHasValidRoles() {
void shouldNotThrowNotAllowedExceptionIfUserHasValidRoles_getComponents() {
User validUser = new User();
validUser.setUserId(USER_ID);
validUser.setRoles(Set.of("RED_USER"));
User validManager = new User();
validManager.setUserId(USER_ID);
validManager.setRoles(Set.of("RED_MANAGER"));
when(componentLogService.getComponentLog(DOSSIER_ID, FILE_ID)).thenReturn(mockComponentLog);
when(mockComponentLog.getComponentLogEntries()).thenReturn(Collections.emptyList());
when(fileStatusService.getFileName(FILE_ID)).thenReturn("mock-file-name");
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser));
assertDoesNotThrow(() -> {
componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false);
});
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validManager));
assertDoesNotThrow(() -> {
componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false);
});
assertDoesNotThrow(() -> componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false));
}
}
@Test
void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_getComponents() {
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager");
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(new User()));
assertThrows(NotAllowedException.class, () -> componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false));
}
@Test
void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_getComponentsOfDossier() {
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager");
assertThrows(NotAllowedException.class, () -> componentControllerV2.getComponentsOfDossier(DOSSIER_TEMPLATE_ID, DOSSIER_ID, false));
}
@Test
void shouldNotThrowExceptionIfApplicationTypeIsValid_getComponentsOfDossier() {
FileStatus fileStatus = FileStatus.builder().fileId(FILE_ID).build();
when(statusController.getDossierStatus(DOSSIER_ID)).thenReturn(Collections.singletonList(fileStatus));
when(componentLogService.getComponentLog(DOSSIER_ID, FILE_ID)).thenReturn(mockComponentLog);
when(fileStatusService.getFileName(FILE_ID)).thenReturn("mock-file-name");
when(mockComponentLog.getComponentLogEntries()).thenReturn(Collections.emptyList());
assertDoesNotThrow(() -> componentControllerV2.getComponentsOfDossier(DOSSIER_TEMPLATE_ID, DOSSIER_ID, false));
}
@Test
void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_addOverride() {
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager");
Component component = Component.builder().name("dummy").componentValues(Collections.emptyList()).build();
assertThrows(NotAllowedException.class, () -> componentControllerV2.addOverride(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, component));
}
@Test
void shouldNotThrowException_addOverride() {
Component component = Component.builder().name("dummy").componentValues(Collections.emptyList()).build();
assertDoesNotThrow(() -> componentControllerV2.addOverride(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, component));
}
@Test
void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_getOverrides() {
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager");
assertThrows(NotAllowedException.class, () -> componentControllerV2.getOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID));
}
@Test
void shouldNotThrowException_getOverrides() {
FileModel fileModel = new FileModel();
fileModel.setId(FILE_ID);
when(fileStatusService.getStatus(FILE_ID)).thenReturn(fileModel);
when(componentLogService.getComponentLog(DOSSIER_ID, FILE_ID)).thenReturn(mockComponentLog);
ComponentLogEntry overriddenEntry = new ComponentLogEntry();
overriddenEntry.setName("dummy");
overriddenEntry.setValues(Collections.emptyList());
overriddenEntry.setOverridden(true);
when(mockComponentLog.getComponentLogEntries()).thenReturn(Collections.singletonList(overriddenEntry));
assertDoesNotThrow(() -> componentControllerV2.getOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID));
}
@Test
void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_revertOverrides() {
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager");
RevertOverrideRequest revertOverrideRequest = new RevertOverrideRequest();
assertThrows(NotAllowedException.class, () -> componentControllerV2.revertOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, revertOverrideRequest));
}
@Test
void shouldNotThrowException_revertOverrides() {
RevertOverrideRequest revertOverrideRequest = new RevertOverrideRequest();
assertDoesNotThrow(() -> componentControllerV2.revertOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, revertOverrideRequest));
}
}

49
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java
View File

@ -4,28 +4,28 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.MockitoAnnotations.openMocks;
import java.io.IOException;
import java.util.HashMap;
import java.lang.reflect.Field;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.util.ReflectionTestUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.persistence.service.v2.external.api.impl.controller.ComponentControllerV2;
import com.iqser.red.service.peristence.v1.server.integration.client.ComponentClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.FileService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.Component;
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentValue;
@ -54,6 +54,17 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest
@Autowired
private FileService fileService;
@Autowired
private ComponentControllerV2 componentControllerV2;
@BeforeEach
@SneakyThrows
void setUp() {
ReflectionTestUtils.setField(componentControllerV2, "applicationType", "DocuMine");
}
@Test
public void testOverrides() throws IOException {
@ -130,21 +141,19 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest
.get(0).isOverridden());
// add and revert override
Component componentOverrideModel3 = Component.builder()
.name("Report_Number")
.componentValues(List.of(ComponentValue.builder()
.value("WOHOO 11/111-111A")
//.originalValue("11/111-111A")
.valueDescription("First found value of type report_number or else ''")
.componentRuleId("ReportNumber.0.0")
.entityReferences(List.of(EntityReference.builder()
.id("e2a93bcc72e9740bbfc19bd9cd982e01")
.type("report_number")
.entityRuleId("DOC.2.0")
.page(1)
.build()))
.build()))
.build();
Component componentOverrideModel3 = Component.builder().name("Report_Number").componentValues(List.of(ComponentValue.builder()
.value("WOHOO 11/111-111A")
//.originalValue("11/111-111A")
.valueDescription(
"First found value of type report_number or else ''")
.componentRuleId("ReportNumber.0.0")
.entityReferences(List.of(EntityReference.builder()
.id("e2a93bcc72e9740bbfc19bd9cd982e01")
.type("report_number")
.entityRuleId("DOC.2.0")
.page(1)
.build()))
.build())).build();
componentClient.addOverride(dossierTemplate.getId(), dossier.getId(), file.getId(), componentOverrideModel3);
overrides = componentClient.getOverrides(dossierTemplate.getId(), dossier.getId(), file.getId());

231
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadControllerV2Test.java Normal file
View File

@ -0,0 +1,231 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import com.iqser.red.persistence.service.v1.external.api.impl.controller.DownloadController;
import com.iqser.red.persistence.service.v2.external.api.impl.controller.DownloadControllerV2;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RemoveDownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
public class DownloadControllerV2Test {
public static final String USER_ID = "user-111";
public static final String DOWNLOAD_ID = "download-123";
@InjectMocks
private DownloadControllerV2 downloadControllerV2;
@Mock
private DownloadController downloadController;
@Mock
private DownloadStatusPersistenceService downloadStatusPersistenceService;
@Mock
private UserService userService;
@Mock
private DownloadStatusEntity mockDownloadStatusEntity;
@BeforeEach
void setUp() {
openMocks(this);
}
@Test
void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_getDownloadStatusList() {
User invalidUser = new User();
invalidUser.setUserId(USER_ID);
invalidUser.setRoles(Set.of("RED_ADMIN"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser));
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertThrows(NotAllowedException.class, () -> downloadControllerV2.getDownloadStatusList());
}
}
@Test
void shouldNotThrowExceptionIfUserHasValidRoles_getDownloadStatusList() {
User validUser = new User();
validUser.setUserId(USER_ID);
validUser.setRoles(Set.of("RED_MANAGER"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser));
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
when(downloadStatusPersistenceService.getStatusesByUser(USER_ID)).thenReturn(Collections.emptyList());
assertDoesNotThrow(() -> downloadControllerV2.getDownloadStatusList());
}
}
@Test
void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_getDownloadStatus() {
User invalidUser = new User();
invalidUser.setUserId(USER_ID);
invalidUser.setRoles(Collections.emptySet());
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser));
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertThrows(NotAllowedException.class, () -> downloadControllerV2.getDownloadStatus(DOWNLOAD_ID));
}
}
@Test
void shouldNotThrowExceptionIfUserHasValidRoles_getDownloadStatus() {
User validUser = new User();
validUser.setUserId(USER_ID);
validUser.setRoles(Set.of("RED_USER"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser));
when(downloadStatusPersistenceService.getStatusesByUuid(DOWNLOAD_ID)).thenReturn(mockDownloadStatusEntity);
when(mockDownloadStatusEntity.getUuid()).thenReturn(DOWNLOAD_ID);
when(mockDownloadStatusEntity.getUserId()).thenReturn(USER_ID);
when(mockDownloadStatusEntity.getFilename()).thenReturn("dummy-file");
when(mockDownloadStatusEntity.getMimeType()).thenReturn("application/pdf");
when(mockDownloadStatusEntity.getErrorCause()).thenReturn(null);
when(mockDownloadStatusEntity.getStatus()).thenReturn(DownloadStatusValue.READY);
when(mockDownloadStatusEntity.getCreationDate()).thenReturn(OffsetDateTime.now());
when(mockDownloadStatusEntity.getLastDownload()).thenReturn(OffsetDateTime.now());
when(mockDownloadStatusEntity.getFileSize()).thenReturn(12345L);
DossierEntity dossierEntity = mock(DossierEntity.class);
when(dossierEntity.getId()).thenReturn("dossier-123");
when(mockDownloadStatusEntity.getDossier()).thenReturn(dossierEntity);
when(mockDownloadStatusEntity.getFiles()).thenReturn(Collections.emptyList());
when(mockDownloadStatusEntity.getDownloadFileTypes()).thenReturn(Collections.emptySet());
when(mockDownloadStatusEntity.getReports()).thenReturn(Collections.emptyList());
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertDoesNotThrow(() -> downloadControllerV2.getDownloadStatus(DOWNLOAD_ID));
}
}
@Test
void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_deleteDownload() {
User invalidUser = new User();
invalidUser.setUserId(USER_ID);
invalidUser.setRoles(Set.of("RED_USER_ADMIN"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser));
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertThrows(NotAllowedException.class, () -> downloadControllerV2.deleteDownload(DOWNLOAD_ID));
}
}
@Test
void shouldNotThrowExceptionIfUserHasValidRoles_deleteDownload() {
User validUser = new User();
validUser.setUserId(USER_ID);
validUser.setRoles(Set.of("RED_MANAGER"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser));
when(downloadStatusPersistenceService.getStatusesByUuid(DOWNLOAD_ID)).thenReturn(mockDownloadStatusEntity);
when(mockDownloadStatusEntity.getStorageId()).thenReturn("storage-123");
doNothing().when(downloadController).deleteDownloadStatus(any(RemoveDownloadRequest.class));
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertDoesNotThrow(() -> downloadControllerV2.deleteDownload(DOWNLOAD_ID));
verify(downloadController).deleteDownloadStatus(any(RemoveDownloadRequest.class));
}
}
@Test
void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_download() {
User invalidUser = new User();
invalidUser.setUserId(USER_ID);
invalidUser.setRoles(Set.of("NON_ELIGIBLE_ROLE"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser));
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertThrows(NotAllowedException.class, () -> downloadControllerV2.download(DOWNLOAD_ID));
}
}
@Test
void shouldNotThrowExceptionIfUserHasValidRoles_download() {
User validUser = new User();
validUser.setUserId(USER_ID);
validUser.setRoles(Set.of("RED_USER"));
when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser));
when(downloadStatusPersistenceService.getStatusesByUuid(DOWNLOAD_ID)).thenReturn(mockDownloadStatusEntity);
when(mockDownloadStatusEntity.getStorageId()).thenReturn("storage-123");
doNothing().when(downloadController).downloadFile("storage-123");
try (MockedStatic<KeycloakSecurity> keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) {
keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID);
assertDoesNotThrow(() -> downloadControllerV2.download(DOWNLOAD_ID));
verify(downloadController).downloadFile("storage-123");
}
}
}

168
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -81,6 +81,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary.
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
@ -3292,6 +3293,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
testRectangleRedactionsBulkLocal(dossierTemplate, dossier, file);
}
private void testBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) {
whenGetEntityLogInvocation();
@ -3341,6 +3343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
String newLegal = "new Legal";
String otherSection = "other section";
String comment1 = "Recategorizing Luke Skywalker37";
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
@ -3349,16 +3352,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.legalBasis(newLegal)
.section(otherSection)
.value("Luke Skywalker37")
.comment(comment1)
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 1);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType(), newType.getType());
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getLegalBasis(), newLegal);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getSection(), otherSection);
assertEquals(1, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(),
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal,
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection,
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getSection());
Long commentId1 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId1);
Optional<CommentEntity> commentEntity1 = commentRepository.findById(commentId1);
assertTrue(commentEntity1.isPresent());
assertEquals(comment1, commentEntity1.get().getText());
String comment2 = "Recategorizing Darth Vader with Legal 3";
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
@ -3368,16 +3382,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section(otherSection)
.value(darthVader)
.originLegalBases(Set.of(legal3))
.comment(comment2)
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 101);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(1).getEntityLogEntry().getType(), newType.getType());
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(29).getEntityLogEntry().getLegalBasis(), newLegal);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(79).getEntityLogEntry().getSection(), otherSection);
assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(),
manualRedactionResponse.getManualAnnotationResponses()
.get(1).getEntityLogEntry().getType());
assertEquals(newLegal,
manualRedactionResponse.getManualAnnotationResponses()
.get(29).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection,
manualRedactionResponse.getManualAnnotationResponses()
.get(79).getEntityLogEntry().getSection());
Long commentId2 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId2);
Optional<CommentEntity> commentEntity2 = commentRepository.findById(commentId2);
assertTrue(commentEntity2.isPresent());
assertEquals(comment2, commentEntity2.get().getText());
String comment3 = "Recategorizing Darth Vader with type2";
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
@ -3387,21 +3412,42 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section(otherSection)
.value(darthVader)
.originTypes(Set.of(type2.getType()))
.comment(comment3)
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 101);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType(), newType.getType());
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(36).getEntityLogEntry().getLegalBasis(), newLegal);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(98).getEntityLogEntry().getSection(), otherSection);
assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(),
manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal,
manualRedactionResponse.getManualAnnotationResponses()
.get(36).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection,
manualRedactionResponse.getManualAnnotationResponses()
.get(98).getEntityLogEntry().getSection());
Long commentId3 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId3);
Optional<CommentEntity> commentEntity3 = commentRepository.findById(commentId3);
assertTrue(commentEntity3.isPresent());
assertEquals(comment3, commentEntity3.get().getText());
String comment4 = "Removing all Darth Vader annotations";
manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(),
file.getId(),
RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).value(darthVader).build(),
RemoveRedactionBulkLocalRequestModel.builder()
.rectangle(false)
.value(darthVader)
.comment(comment4)
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 202);
assertEquals(202, manualRedactionResponse.getManualAnnotationResponses().size());
Long commentId4 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId4);
Optional<CommentEntity> commentEntity4 = commentRepository.findById(commentId4);
assertTrue(commentEntity4.isPresent());
assertEquals(comment4, commentEntity4.get().getText());
}
@ -3728,6 +3774,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertEquals(EntryState.APPLIED, rectangleAnnotationEntry.getState());
String awesomeComment = "awesome comment";
String myUser = "MyUser";
BulkLocalResponse addValueBackResponse = BulkLocalResponse.builder()
.fileId(file.getId())
.dossierId(dossier.getId())
@ -3737,6 +3784,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.section(section)
.foundTerms(List.of(new FoundTerm(List.of(position), value)))
.comment(awesomeComment)
.userId(myUser)
.build();
searchTermOccurrencesResponseReceiver.receive(addValueBackResponse);
@ -3761,6 +3809,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
List<CommentEntity> commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), newValueAnnotation.getId(), false);
assertEquals(commentEntities.size(), 1);
assertEquals(commentEntities.get(0).getText(), awesomeComment);
assertEquals(commentEntities.get(0).getUser(), myUser);
removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, true);
@ -3803,6 +3852,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), appliedValueAnnotations.get(0).getId(), false);
assertEquals(commentEntities.size(), 1);
assertEquals(commentEntities.get(0).getText(), awesomeComment);
assertEquals(commentEntities.get(0).getUser(), myUser);
String totallyDifferentComment = "totally different comment";
RemoveRedactionBulkLocalRequestModel removeRectangleRequest = RemoveRedactionBulkLocalRequestModel.builder()
@ -3884,6 +3934,78 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
}
@Test
public void testLocalAddWithRectangle() {
whenGetEntityLogInvocation();
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var type = typeProvider.testAndProvideType(dossierTemplate, dossier, "test", false, 100);
var entityLog = new EntityLog(1, 1, new ArrayList<>(), null, 0, 0, 0, 0);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
AddRedactionBulkLocalRequestModel addRedactionRequest = AddRedactionBulkLocalRequestModel.builder()
.type(type.getType())
.value("non-readable content")
.legalBasis("legal basis")
.reason("reason")
.section("section")
.comment("test comment")
.positions(List.of(new Rectangle(100f, 200f, 50f, 50f, 0)))
.pageNumbers(Set.of(1, 2, 3))
.rectangle(true)
.build();
ManualRedactionResponse response = manualRedactionClient.addRedactionBulkLocal(dossier.getId(), file.getId(), addRedactionRequest);
assertNotNull(response);
assertEquals(3, response.getManualAnnotationResponses().size());
EntityLog currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId());
assertNotNull(currentEntityLog);
assertEquals(3, currentEntityLog.getEntityLogEntry().size());
for (int i = 1; i <= 3; i++) {
int page = i;
EntityLogEntry entry = currentEntityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getPositions() != null
&& !e.getPositions().isEmpty()
&& e.getPositions()
.get(0).getPageNumber() == page)
.findFirst()
.orElse(null);
assertNotNull(entry);
assertEquals("non-readable content", entry.getValue());
assertEquals("legal basis", entry.getLegalBasis());
assertEquals("reason", entry.getReason());
assertEquals("section", entry.getSection());
assertEquals(type.getType(), entry.getType());
assertEquals(EntryType.AREA, entry.getEntryType());
assertEquals(EntryState.APPLIED, entry.getState());
assertEquals(1, entry.getPositions().size());
Position position = entry.getPositions()
.get(0);
assertEquals(page, position.getPageNumber());
assertEquals(100f, position.x());
assertEquals(200f, position.y());
assertEquals(50f, position.w());
assertEquals(50f, position.h());
}
for (EntityLogEntry entry : currentEntityLog.getEntityLogEntry()) {
List<CommentEntity> comments = commentRepository.findByFileIdAndAnnotationId(file.getId(), entry.getId(), false);
assertEquals(1, comments.size());
CommentEntity comment = comments.get(0);
assertEquals("test comment", comment.getText());
}
}
private Rectangle toRectangle(Position position) {
return new Rectangle(position.x(), position.y(), position.w(), position.h(), 0);

1
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java
View File

@ -69,6 +69,7 @@ public class SearchTermOccurrencesTest extends AbstractPersistenceServerServiceT
.section("section")
.foundTerms(List.of(new FoundTerm(List.of(new Position(new float[]{1f, 2f, 3f, 4f}, 1)), "searchTerm")))
.comment(null)
.userId("MyUser")
.build());
List<ManualRedactionEntryEntity> newEntries = manualRedactionRepository.findByFileIdAndOptions(file.getId(), false, false, false);

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java
View File

@ -36,4 +36,7 @@ public class BulkLocalRequest {
private String comment;
@NonNull
private String userId;
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java
View File

@ -39,4 +39,6 @@ public class BulkLocalResponse {
private String comment;
@NonNull
private String userId;
}

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java
View File

@ -7,5 +7,6 @@ public enum DownloadFileType {
REDACTED,
ANNOTATED,
FLATTEN,
DELTA_PREVIEW
DELTA_PREVIEW,
DATASET_EXPORT
}

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java
View File

@ -35,6 +35,8 @@ public class AddRedactionBulkLocalRequestModel {
private String section;
private boolean rectangle;
@NonNull
@Builder.Default
private List<Rectangle> positions = new ArrayList<>();