From 9b07def91d382568d922cb0c7643ff0b075a05d7 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Wed, 9 Oct 2024 14:40:20 +0200 Subject: [PATCH 1/4] RED-10112: Wrong role permissions in API endpoints for GUEST and RED_USER_ADMIN role --- .../controller/ComponentControllerV2.java | 32 ++- .../impl/controller/DownloadControllerV2.java | 24 ++ .../v1/processor/roles/ApplicationRoles.java | 12 +- .../tests/ComponentControllerV2Test.java | 161 +++++++++--- .../tests/ComponentOverrideTest.java | 49 ++-- .../tests/DownloadControllerV2Test.java | 231 ++++++++++++++++++ 6 files changed, 449 insertions(+), 60 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadControllerV2Test.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java index 6383422e3..9240293d8 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/ComponentControllerV2.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.Optional; import java.util.stream.Collectors; +import org.springframework.beans.factory.annotation.Value; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -42,15 +43,17 @@ import lombok.experimental.FieldDefaults; @RestController @RequiredArgsConstructor @Tag(name = "4. Component endpoints", description = "Provides operations related to components") -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ComponentControllerV2 implements ComponentResource { - ComponentLogService componentLogService; - UserService userService; - StatusController statusController; - FileStatusService fileStatusService; - DossierTemplatePersistenceService dossierTemplatePersistenceService; - ComponentMapper componentMapper = ComponentMapper.INSTANCE; + private final ComponentLogService componentLogService; + private final UserService userService; + private final StatusController statusController; + private final FileStatusService fileStatusService; + private final DossierTemplatePersistenceService dossierTemplatePersistenceService; + private final ComponentMapper componentMapper = ComponentMapper.INSTANCE; + + @Value("${application.type}") + private String applicationType; @Override @@ -59,8 +62,9 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(FILE_ID_PARAM) String fileId, @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) { - dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); + checkApplicationType(); validateUserRoles(KeycloakSecurity.getUserId()); + dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); var componentLog = componentLogService.getComponentLog(dossierId, fileId); return componentMapper.toFileComponents(componentLog, dossierTemplateId, dossierId, fileId, fileStatusService.getFileName(fileId), includeDetails); @@ -85,6 +89,7 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(name = INCLUDE_DETAILS_PARAM, defaultValue = "false", required = false) boolean includeDetails) { + checkApplicationType(); dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); var dossierFiles = statusController.getDossierStatus(dossierId); return new FileComponentsList(dossierFiles.stream() @@ -100,6 +105,7 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(FILE_ID_PARAM) String fileId, @RequestBody Component override) { + checkApplicationType(); dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); componentLogService.overrideComponent(dossierId, fileId, componentMapper.toComponentLogEntry(override)); @@ -112,6 +118,7 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId) { + checkApplicationType(); dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); FileModel status = fileStatusService.getStatus(fileId); @@ -138,9 +145,18 @@ public class ComponentControllerV2 implements ComponentResource { @PathVariable(FILE_ID_PARAM) String fileId, @RequestBody RevertOverrideRequest revertOverrideRequest) { + checkApplicationType(); dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); componentLogService.revertOverrides(dossierId, fileId, revertOverrideRequest); } + + private void checkApplicationType() { + + if(!applicationType.equals("DocuMine")) { + throw new NotAllowedException("Components can only be accessed in DocuMine"); + } + } + } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java index 5a89c5332..4579b2c13 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DownloadControllerV2.java @@ -1,6 +1,7 @@ package com.iqser.red.persistence.service.v2.external.api.impl.controller; import java.util.List; +import java.util.Optional; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; @@ -8,7 +9,11 @@ import org.springframework.web.bind.annotation.RestController; import com.iqser.red.persistence.service.v1.external.api.impl.controller.DownloadController; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; +import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User; import com.iqser.red.service.persistence.service.v1.api.shared.model.RemoveDownloadRequest; import com.iqser.red.service.persistence.service.v2.api.external.model.DownloadStatus; import com.iqser.red.service.persistence.service.v2.api.external.model.DownloadStatusList; @@ -26,10 +31,13 @@ public class DownloadControllerV2 implements DownloadResource { private final DownloadController downloadController; private final DownloadStatusPersistenceService downloadStatusPersistenceService; + private final UserService userService; + @Transactional public DownloadStatusList getDownloadStatusList() { + validateUserRoles(KeycloakSecurity.getUserId()); var downloads = downloadStatusPersistenceService.getStatusesByUser(KeycloakSecurity.getUserId()); return new DownloadStatusList(downloads.stream().map( @@ -61,6 +69,7 @@ public class DownloadControllerV2 implements DownloadResource { @Transactional public DownloadStatus getDownloadStatus(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) { + validateUserRoles(KeycloakSecurity.getUserId()); var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId); return DownloadStatus.builder() @@ -88,6 +97,7 @@ public class DownloadControllerV2 implements DownloadResource { public void deleteDownload(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) { + validateUserRoles(KeycloakSecurity.getUserId()); var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId); downloadController.deleteDownloadStatus(new RemoveDownloadRequest(List.of(status.getStorageId()))); } @@ -95,8 +105,22 @@ public class DownloadControllerV2 implements DownloadResource { public void download(@PathVariable(DOWNLOAD_ID_PARAM) String downloadId) { + validateUserRoles(KeycloakSecurity.getUserId()); var status = downloadStatusPersistenceService.getStatusesByUuid(downloadId); downloadController.downloadFile(status.getStorageId()); } + + private void validateUserRoles(String userId) { + + Optional userOptional = userService.getUserById(userId); + if (userOptional.isPresent()) { + if (userOptional.get().getRoles() + .stream() + .noneMatch(ApplicationRoles.VALID_MEMBER_ROLES::contains)) { + throw new NotAllowedException("User doesn't have appropriate roles"); + } + } + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java index b9856f9c3..05ced6d54 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ApplicationRoles.java @@ -53,7 +53,9 @@ public final class ApplicationRoles { DOWNLOAD_REDACTION_PREVIEW_FILE, DOWNLOAD_REPORT_TEMPLATE, EXCLUDE_INCLUDE_FILE, - EXCLUDE_INCLUDE_PAGES, GET_REPORT_TEMPLATES, GET_SIMILAR_IMAGES, + EXCLUDE_INCLUDE_PAGES, + GET_REPORT_TEMPLATES, + GET_SIMILAR_IMAGES, MANAGE_USER_PREFERENCES, MANAGE_VIEWED_PAGES, PROCESS_DOWNLOAD, @@ -105,7 +107,9 @@ public final class ApplicationRoles { DELETE_DICTIONARY_TYPE, DELETE_REPORT_TEMPLATE, DOWNLOAD_REPORT_TEMPLATE, - GET_REPORT_TEMPLATES, MANAGE_USER_PREFERENCES, GET_SIMILAR_IMAGES, + GET_REPORT_TEMPLATES, + MANAGE_USER_PREFERENCES, + GET_SIMILAR_IMAGES, READ_COLORS, READ_DICTIONARY_TYPES, READ_DIGITAL_SIGNATURE, @@ -147,7 +151,9 @@ public final class ApplicationRoles { public static final Set RED_USER_ADMIN_ACTION_ROLES = Sets.newHashSet(MANAGE_USER_PREFERENCES, READ_ALL_USERS, READ_APP_CONFIG, - READ_GENERAL_CONFIGURATION, READ_GENERAL_CONFIGURATION, GET_SIMILAR_IMAGES, + READ_GENERAL_CONFIGURATION, + READ_GENERAL_CONFIGURATION, + GET_SIMILAR_IMAGES, READ_NOTIFICATIONS, READ_USERS, UPDATE_MY_PROFILE, diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentControllerV2Test.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentControllerV2Test.java index e599aa426..46e110a4e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentControllerV2Test.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentControllerV2Test.java @@ -1,18 +1,21 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; + import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.openMocks; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; -import org.mockito.MockitoAnnotations; - -import java.util.Collections; -import java.util.Optional; -import java.util.Set; +import org.springframework.test.util.ReflectionTestUtils; import com.iqser.red.persistence.service.v1.external.api.impl.controller.StatusController; import com.iqser.red.persistence.service.v2.external.api.impl.controller.ComponentControllerV2; @@ -22,17 +25,26 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; +import com.iqser.red.service.persistence.service.v2.api.external.model.Component; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; +import lombok.SneakyThrows; + public class ComponentControllerV2Test { public static final String USER_ID = "user-111"; public static final String FILE_ID = "file-789"; public static final String DOSSIER_ID = "dossier-456"; public static final String DOSSIER_TEMPLATE_ID = "template-123"; + @InjectMocks private ComponentControllerV2 componentControllerV2; + @Mock private UserService userService; @Mock @@ -43,17 +55,22 @@ public class ComponentControllerV2Test { private DossierTemplatePersistenceService dossierTemplatePersistenceService; @Mock private StatusController statusController; - @Mock private ComponentLog mockComponentLog; + @BeforeEach + @SneakyThrows void setUp() { - MockitoAnnotations.openMocks(this); + + openMocks(this); + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "DocuMine"); } + @Test - void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles() { + void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_getComponents() { User invalidUser = new User(); invalidUser.setUserId(USER_ID); @@ -64,43 +81,129 @@ public class ComponentControllerV2Test { try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); - assertThrows(NotAllowedException.class, () -> { - componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false); - }); + assertThrows(NotAllowedException.class, () -> componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false)); } } - @Test - void shouldNotThrowNotAllowedExceptionIfUserHasValidRoles() { - + void shouldNotThrowNotAllowedExceptionIfUserHasValidRoles_getComponents() { User validUser = new User(); validUser.setUserId(USER_ID); validUser.setRoles(Set.of("RED_USER")); - User validManager = new User(); - validManager.setUserId(USER_ID); - validManager.setRoles(Set.of("RED_MANAGER")); - when(componentLogService.getComponentLog(DOSSIER_ID, FILE_ID)).thenReturn(mockComponentLog); - when(mockComponentLog.getComponentLogEntries()).thenReturn(Collections.emptyList()); - when(fileStatusService.getFileName(FILE_ID)).thenReturn("mock-file-name"); try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); - when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser)); - assertDoesNotThrow(() -> { - componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false); - }); - when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validManager)); - assertDoesNotThrow(() -> { - componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false); - }); + + assertDoesNotThrow(() -> componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false)); } } + + + @Test + void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_getComponents() { + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager"); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(new User())); + + assertThrows(NotAllowedException.class, () -> componentControllerV2.getComponents(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, false)); + } + + + @Test + void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_getComponentsOfDossier() { + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager"); + + assertThrows(NotAllowedException.class, () -> componentControllerV2.getComponentsOfDossier(DOSSIER_TEMPLATE_ID, DOSSIER_ID, false)); + } + + + @Test + void shouldNotThrowExceptionIfApplicationTypeIsValid_getComponentsOfDossier() { + + FileStatus fileStatus = FileStatus.builder().fileId(FILE_ID).build(); + when(statusController.getDossierStatus(DOSSIER_ID)).thenReturn(Collections.singletonList(fileStatus)); + + when(componentLogService.getComponentLog(DOSSIER_ID, FILE_ID)).thenReturn(mockComponentLog); + when(fileStatusService.getFileName(FILE_ID)).thenReturn("mock-file-name"); + when(mockComponentLog.getComponentLogEntries()).thenReturn(Collections.emptyList()); + + assertDoesNotThrow(() -> componentControllerV2.getComponentsOfDossier(DOSSIER_TEMPLATE_ID, DOSSIER_ID, false)); + } + + + @Test + void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_addOverride() { + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager"); + + Component component = Component.builder().name("dummy").componentValues(Collections.emptyList()).build(); + + assertThrows(NotAllowedException.class, () -> componentControllerV2.addOverride(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, component)); + } + + + @Test + void shouldNotThrowException_addOverride() { + + Component component = Component.builder().name("dummy").componentValues(Collections.emptyList()).build(); + + assertDoesNotThrow(() -> componentControllerV2.addOverride(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, component)); + } + + + @Test + void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_getOverrides() { + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager"); + + assertThrows(NotAllowedException.class, () -> componentControllerV2.getOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID)); + } + + + @Test + void shouldNotThrowException_getOverrides() { + + FileModel fileModel = new FileModel(); + fileModel.setId(FILE_ID); + when(fileStatusService.getStatus(FILE_ID)).thenReturn(fileModel); + + when(componentLogService.getComponentLog(DOSSIER_ID, FILE_ID)).thenReturn(mockComponentLog); + ComponentLogEntry overriddenEntry = new ComponentLogEntry(); + overriddenEntry.setName("dummy"); + overriddenEntry.setValues(Collections.emptyList()); + overriddenEntry.setOverridden(true); + when(mockComponentLog.getComponentLogEntries()).thenReturn(Collections.singletonList(overriddenEntry)); + + assertDoesNotThrow(() -> componentControllerV2.getOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID)); + } + + + @Test + void shouldThrowNotAllowedExceptionIfApplicationTypeIsRM_revertOverrides() { + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "RedactManager"); + + RevertOverrideRequest revertOverrideRequest = new RevertOverrideRequest(); + + assertThrows(NotAllowedException.class, () -> componentControllerV2.revertOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, revertOverrideRequest)); + } + + + @Test + void shouldNotThrowException_revertOverrides() { + + RevertOverrideRequest revertOverrideRequest = new RevertOverrideRequest(); + + assertDoesNotThrow(() -> componentControllerV2.revertOverrides(DOSSIER_TEMPLATE_ID, DOSSIER_ID, FILE_ID, revertOverrideRequest)); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java index 97819563e..8e79a863f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ComponentOverrideTest.java @@ -4,28 +4,28 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.MockitoAnnotations.openMocks; import java.io.IOException; -import java.util.HashMap; +import java.lang.reflect.Field; import java.util.List; -import java.util.Map; import java.util.Set; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; +import org.springframework.test.util.ReflectionTestUtils; import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.persistence.service.v2.external.api.impl.controller.ComponentControllerV2; import com.iqser.red.service.peristence.v1.server.integration.client.ComponentClient; import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.FileService; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentsOverrides; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.RevertOverrideRequest; import com.iqser.red.service.persistence.service.v2.api.external.model.Component; import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentValue; @@ -54,6 +54,17 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest @Autowired private FileService fileService; + @Autowired + private ComponentControllerV2 componentControllerV2; + + + @BeforeEach + @SneakyThrows + void setUp() { + + ReflectionTestUtils.setField(componentControllerV2, "applicationType", "DocuMine"); + } + @Test public void testOverrides() throws IOException { @@ -130,21 +141,19 @@ public class ComponentOverrideTest extends AbstractPersistenceServerServiceTest .get(0).isOverridden()); // add and revert override - Component componentOverrideModel3 = Component.builder() - .name("Report_Number") - .componentValues(List.of(ComponentValue.builder() - .value("WOHOO 11/111-111A") - //.originalValue("11/111-111A") - .valueDescription("First found value of type report_number or else ''") - .componentRuleId("ReportNumber.0.0") - .entityReferences(List.of(EntityReference.builder() - .id("e2a93bcc72e9740bbfc19bd9cd982e01") - .type("report_number") - .entityRuleId("DOC.2.0") - .page(1) - .build())) - .build())) - .build(); + Component componentOverrideModel3 = Component.builder().name("Report_Number").componentValues(List.of(ComponentValue.builder() + .value("WOHOO 11/111-111A") + //.originalValue("11/111-111A") + .valueDescription( + "First found value of type report_number or else ''") + .componentRuleId("ReportNumber.0.0") + .entityReferences(List.of(EntityReference.builder() + .id("e2a93bcc72e9740bbfc19bd9cd982e01") + .type("report_number") + .entityRuleId("DOC.2.0") + .page(1) + .build())) + .build())).build(); componentClient.addOverride(dossierTemplate.getId(), dossier.getId(), file.getId(), componentOverrideModel3); overrides = componentClient.getOverrides(dossierTemplate.getId(), dossier.getId(), file.getId()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadControllerV2Test.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadControllerV2Test.java new file mode 100644 index 000000000..a94bffa1d --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadControllerV2Test.java @@ -0,0 +1,231 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.openMocks; + +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; + +import com.iqser.red.persistence.service.v1.external.api.impl.controller.DownloadController; +import com.iqser.red.persistence.service.v2.external.api.impl.controller.DownloadControllerV2; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; +import com.iqser.red.service.persistence.management.v1.processor.service.users.model.User; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RemoveDownloadRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; +import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; + +public class DownloadControllerV2Test { + + public static final String USER_ID = "user-111"; + public static final String DOWNLOAD_ID = "download-123"; + + @InjectMocks + private DownloadControllerV2 downloadControllerV2; + + @Mock + private DownloadController downloadController; + @Mock + private DownloadStatusPersistenceService downloadStatusPersistenceService; + @Mock + private UserService userService; + + @Mock + private DownloadStatusEntity mockDownloadStatusEntity; + + + @BeforeEach + void setUp() { + + openMocks(this); + } + + + @Test + void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_getDownloadStatusList() { + + User invalidUser = new User(); + invalidUser.setUserId(USER_ID); + invalidUser.setRoles(Set.of("RED_ADMIN")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser)); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertThrows(NotAllowedException.class, () -> downloadControllerV2.getDownloadStatusList()); + } + } + + + @Test + void shouldNotThrowExceptionIfUserHasValidRoles_getDownloadStatusList() { + + User validUser = new User(); + validUser.setUserId(USER_ID); + validUser.setRoles(Set.of("RED_MANAGER")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser)); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + when(downloadStatusPersistenceService.getStatusesByUser(USER_ID)).thenReturn(Collections.emptyList()); + + assertDoesNotThrow(() -> downloadControllerV2.getDownloadStatusList()); + } + } + + + @Test + void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_getDownloadStatus() { + + User invalidUser = new User(); + invalidUser.setUserId(USER_ID); + invalidUser.setRoles(Collections.emptySet()); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser)); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertThrows(NotAllowedException.class, () -> downloadControllerV2.getDownloadStatus(DOWNLOAD_ID)); + } + } + + + @Test + void shouldNotThrowExceptionIfUserHasValidRoles_getDownloadStatus() { + + User validUser = new User(); + validUser.setUserId(USER_ID); + validUser.setRoles(Set.of("RED_USER")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser)); + + when(downloadStatusPersistenceService.getStatusesByUuid(DOWNLOAD_ID)).thenReturn(mockDownloadStatusEntity); + + when(mockDownloadStatusEntity.getUuid()).thenReturn(DOWNLOAD_ID); + when(mockDownloadStatusEntity.getUserId()).thenReturn(USER_ID); + when(mockDownloadStatusEntity.getFilename()).thenReturn("dummy-file"); + when(mockDownloadStatusEntity.getMimeType()).thenReturn("application/pdf"); + when(mockDownloadStatusEntity.getErrorCause()).thenReturn(null); + when(mockDownloadStatusEntity.getStatus()).thenReturn(DownloadStatusValue.READY); + when(mockDownloadStatusEntity.getCreationDate()).thenReturn(OffsetDateTime.now()); + when(mockDownloadStatusEntity.getLastDownload()).thenReturn(OffsetDateTime.now()); + when(mockDownloadStatusEntity.getFileSize()).thenReturn(12345L); + DossierEntity dossierEntity = mock(DossierEntity.class); + when(dossierEntity.getId()).thenReturn("dossier-123"); + when(mockDownloadStatusEntity.getDossier()).thenReturn(dossierEntity); + when(mockDownloadStatusEntity.getFiles()).thenReturn(Collections.emptyList()); + when(mockDownloadStatusEntity.getDownloadFileTypes()).thenReturn(Collections.emptySet()); + when(mockDownloadStatusEntity.getReports()).thenReturn(Collections.emptyList()); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertDoesNotThrow(() -> downloadControllerV2.getDownloadStatus(DOWNLOAD_ID)); + } + } + + + @Test + void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_deleteDownload() { + + User invalidUser = new User(); + invalidUser.setUserId(USER_ID); + invalidUser.setRoles(Set.of("RED_USER_ADMIN")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser)); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertThrows(NotAllowedException.class, () -> downloadControllerV2.deleteDownload(DOWNLOAD_ID)); + } + } + + + @Test + void shouldNotThrowExceptionIfUserHasValidRoles_deleteDownload() { + + User validUser = new User(); + validUser.setUserId(USER_ID); + validUser.setRoles(Set.of("RED_MANAGER")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser)); + + when(downloadStatusPersistenceService.getStatusesByUuid(DOWNLOAD_ID)).thenReturn(mockDownloadStatusEntity); + when(mockDownloadStatusEntity.getStorageId()).thenReturn("storage-123"); + + doNothing().when(downloadController).deleteDownloadStatus(any(RemoveDownloadRequest.class)); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertDoesNotThrow(() -> downloadControllerV2.deleteDownload(DOWNLOAD_ID)); + + verify(downloadController).deleteDownloadStatus(any(RemoveDownloadRequest.class)); + } + } + + + @Test + void shouldThrowNotAllowedExceptionIfUserHasInvalidRoles_download() { + + User invalidUser = new User(); + invalidUser.setUserId(USER_ID); + invalidUser.setRoles(Set.of("NON_ELIGIBLE_ROLE")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(invalidUser)); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertThrows(NotAllowedException.class, () -> downloadControllerV2.download(DOWNLOAD_ID)); + } + } + + + @Test + void shouldNotThrowExceptionIfUserHasValidRoles_download() { + + User validUser = new User(); + validUser.setUserId(USER_ID); + validUser.setRoles(Set.of("RED_USER")); + + when(userService.getUserById(USER_ID)).thenReturn(Optional.of(validUser)); + + when(downloadStatusPersistenceService.getStatusesByUuid(DOWNLOAD_ID)).thenReturn(mockDownloadStatusEntity); + when(mockDownloadStatusEntity.getStorageId()).thenReturn("storage-123"); + + doNothing().when(downloadController).downloadFile("storage-123"); + + try (MockedStatic keycloakSecurityMock = mockStatic(KeycloakSecurity.class)) { + keycloakSecurityMock.when(KeycloakSecurity::getUserId).thenReturn(USER_ID); + + assertDoesNotThrow(() -> downloadControllerV2.download(DOWNLOAD_ID)); + + verify(downloadController).downloadFile("storage-123"); + } + } + +} From 82661dcaf2912b68f9f102e1d8e67dec2a0524c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 10 Oct 2024 11:20:33 +0200 Subject: [PATCH 2/4] RED-10149: Do not re- ocr file after pages rotations; RED-10083: Minor fix in openapi spec --- .../api/impl/controller/FileManagementController.java | 11 +++++++---- .../src/main/resources/api/documine.yaml | 6 +++--- .../src/main/resources/api/redactmanager.yaml | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java index 7fba63821..2dfc208cf 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileManagementController.java @@ -266,11 +266,14 @@ public class FileManagementController implements FileManagementResource { FileModel fileModel = fileStatusManagementService.getFileStatus(fileId); if (!fileModel.isExcludedFromAutomaticAnalysis()) { - if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) { - reanalysisService.ocrFile(dossierId, fileId, true); - } else { + + // Re- ocr is files with pdftron ocr service is breaking files and not needed with azure. + // TODO find a better solution for this. +// if (fileModel.getOcrStartTime() != null || fileModel.getOcrEndTime() != null) { +// reanalysisService.ocrFile(dossierId, fileId, true); +// } else { fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true); - } +// } } auditPersistenceService.audit(AuditRequest.builder() diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml index c1b93f59e..a389ae69d 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml @@ -3629,7 +3629,7 @@ components: modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6 validFrom: 2020-01-01T00:00:00.000+00:00 validTo: 2030-12-31T23:59:59.999+00:00 - dossierTemplateStatus: ACTIVE + status: ACTIVE removeWatermark: false keepImageMetadata: true ocrByDefault: true @@ -3750,7 +3750,7 @@ components: modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6 validFrom: 2020-01-01T00:00:00.000+00:00 validTo: 2030-12-31T23:59:59.999+00:00 - dossierTemplateStatus: ACTIVE + status: ACTIVE removeWatermark: false keepImageMetadata: true ocrByDefault: true @@ -3768,7 +3768,7 @@ components: modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d validFrom: 2023-01-01T00:00:00.000+00:00 validTo: 2033-12-31T23:59:59.999+00:00 - dossierTemplateStatus: ACTIVE + status: ACTIVE removeWatermark: true keepImageMetadata: true ocrByDefault: true diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml index a8ac8f81d..9b58d7b6f 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/redactmanager.yaml @@ -2076,7 +2076,7 @@ components: modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6 validFrom: 2020-01-01T00:00:00.000+00:00 validTo: 2030-12-31T23:59:59.999+00:00 - dossierTemplateStatus: ACTIVE + status: ACTIVE removeWatermark: false keepImageMetadata: false ocrByDefault: false @@ -2212,7 +2212,7 @@ components: modifiedBy: c2e33246-e50a-4c43-831c-6789a5637db6 validFrom: 2020-01-01T00:00:00.000+00:00 validTo: 2030-12-31T23:59:59.999+00:00 - dossierTemplateStatus: ACTIVE + status: ACTIVE removeWatermark: false keepImageMetadata: false ocrByDefault: false @@ -2231,7 +2231,7 @@ components: modifiedBy: 46a7f9d3-6ba0-41d7-b312-b8e708aa6f4d validFrom: 2023-01-01T00:00:00.000+00:00 validTo: 2033-12-31T23:59:59.999+00:00 - dossierTemplateStatus: ACTIVE + status: ACTIVE removeWatermark: true keepImageMetadata: false ocrByDefault: true From cdbf69abbf7639e407b7a7a94e0c4122d9d0247e Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Mon, 14 Oct 2024 14:30:46 +0200 Subject: [PATCH 3/4] RED-10104 && RED-10178 --- .../controller/ManualRedactionController.java | 42 +++- .../processor/service/FileStatusService.java | 2 + ...SearchTermOccurrencesResponseReceiver.java | 10 + .../tests/ManualRedactionTest.java | 214 ++++++++++++------ .../tests/SearchTermOccurrencesTest.java | 1 + .../v1/api/shared/model/BulkLocalRequest.java | 3 + .../api/shared/model/BulkLocalResponse.java | 2 + .../AddRedactionBulkLocalRequestModel.java | 2 + 8 files changed, 201 insertions(+), 75 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java index ac4b34968..93baa6b1f 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ManualRedactionController.java @@ -46,6 +46,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactionResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel; @@ -211,17 +212,40 @@ public class ManualRedactionController implements ManualRedactionResource { var dossier = dossierManagementService.getDossierById(dossierId, false, false); dictionaryPersistenceService.getType(TypeIdUtils.toTypeId(addRedactionRequest.getType(), dossier.getDossierTemplateId())); - fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest); + if (!addRedactionRequest.isRectangle()) { + fileStatusService.setStatusBulkLocalRedactionsProcessing(fileId, addRedactionRequest); - EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest); - - return ManualRedactionResponse.builder() - .manualAnnotationResponses(List.of(ManualAnnotationResponse.builder() - .annotationId(manualRedactionService.getNewAnnotationId()) - .entityLogEntry(entityLogEntry) - .build())) - .build(); + EntityLogEntry entityLogEntry = pendingEntryFactory.buildAddRedactionBulkLocalEntry(addRedactionRequest); + return ManualRedactionResponse.builder() + .manualAnnotationResponses(List.of(ManualAnnotationResponse.builder() + .annotationId(manualRedactionService.getNewAnnotationId()) + .entityLogEntry(entityLogEntry) + .build())) + .build(); + } else { + Set addRedactionRequestModels = addRedactionRequest.getPageNumbers() + .stream() + .map(page -> AddRedactionRequestModel.builder() + .type(addRedactionRequest.getType()) + .value(addRedactionRequest.getValue()) + .reason(addRedactionRequest.getReason()) + .legalBasis(addRedactionRequest.getLegalBasis()) + .positions(addRedactionRequest.getPositions() + .stream() + .map(rectangle -> Rectangle.createRectangle(rectangle.getTopLeftX(), + rectangle.getTopLeftY(), + rectangle.getWidth(), + rectangle.getHeight(), + page)) + .toList()) + .comment(addRedactionRequest.getComment() != null ? AddCommentRequestModel.builder().text(addRedactionRequest.getComment()).build() : null) + .section(addRedactionRequest.getSection()) + .rectangle(true) + .build()) + .collect(Collectors.toSet()); + return addRedactionBulk(dossierId, fileId, addRedactionRequestModels); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 630f7b80f..bfeab9e79 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -73,6 +73,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddR import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; +import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import com.knecon.fforesight.llm.service.LlmNerMessage; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest; @@ -1060,6 +1061,7 @@ public class FileStatusService { .section(addRedactionBulkLocalRequestModel.getSection()) .pageNumbers(addRedactionBulkLocalRequestModel.getPageNumbers()) .comment(addRedactionBulkLocalRequestModel.getComment()) + .userId(KeycloakSecurity.getUserId()) .build(); addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java index 0ba4d4c5b..dd2837135 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java @@ -10,6 +10,11 @@ import org.springframework.amqp.AmqpRejectAndDontRequeueException; import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitHandler; import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.context.SecurityContextImpl; import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.type.TypeReference; @@ -73,6 +78,11 @@ public class SearchTermOccurrencesResponseReceiver { public void receive(BulkLocalResponse response) { + Authentication authentication = new UsernamePasswordAuthenticationToken(response.getUserId(), null, null); + SecurityContextImpl securityContext = new SecurityContextImpl(); + securityContext.setAuthentication(authentication); + SecurityContextHolder.setContext(securityContext); + var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false); Set addRedactionRequests = response.getFoundTerms() diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index fbf067342..131664aa2 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -81,6 +81,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dictionary. import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionBulkLocalRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel; @@ -3292,6 +3293,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { testRectangleRedactionsBulkLocal(dossierTemplate, dossier, file); } + private void testBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) { whenGetEntityLogInvocation(); @@ -3342,25 +3344,27 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { String newLegal = "new Legal"; String otherSection = "other section"; String comment1 = "Recategorizing Luke Skywalker37"; - ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal( - dossier.getId(), - file.getId(), - RecategorizationBulkLocalRequestModel.builder() - .rectangle(false) - .type(newType.getType()) - .legalBasis(newLegal) - .section(otherSection) - .value("Luke Skywalker37") - .comment(comment1) - .build(), - false); + ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), + file.getId(), + RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis(newLegal) + .section(otherSection) + .value("Luke Skywalker37") + .comment(comment1) + .build(), + false); assertEquals(1, manualRedactionResponse.getManualAnnotationResponses().size()); - assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses() - .get(0).getEntityLogEntry().getType()); - assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses() - .get(0).getEntityLogEntry().getLegalBasis()); - assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses() - .get(0).getEntityLogEntry().getSection()); + assertEquals(newType.getType(), + manualRedactionResponse.getManualAnnotationResponses() + .get(0).getEntityLogEntry().getType()); + assertEquals(newLegal, + manualRedactionResponse.getManualAnnotationResponses() + .get(0).getEntityLogEntry().getLegalBasis()); + assertEquals(otherSection, + manualRedactionResponse.getManualAnnotationResponses() + .get(0).getEntityLogEntry().getSection()); Long commentId1 = manualRedactionResponse.getManualAnnotationResponses() .get(0).getCommentId(); assertNotNull(commentId1); @@ -3369,26 +3373,28 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertEquals(comment1, commentEntity1.get().getText()); String comment2 = "Recategorizing Darth Vader with Legal 3"; - manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal( - dossier.getId(), - file.getId(), - RecategorizationBulkLocalRequestModel.builder() - .rectangle(false) - .type(newType.getType()) - .legalBasis(newLegal) - .section(otherSection) - .value(darthVader) - .originLegalBases(Set.of(legal3)) - .comment(comment2) - .build(), - false); + manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), + file.getId(), + RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis(newLegal) + .section(otherSection) + .value(darthVader) + .originLegalBases(Set.of(legal3)) + .comment(comment2) + .build(), + false); assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size()); - assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses() - .get(1).getEntityLogEntry().getType()); - assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses() - .get(29).getEntityLogEntry().getLegalBasis()); - assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses() - .get(79).getEntityLogEntry().getSection()); + assertEquals(newType.getType(), + manualRedactionResponse.getManualAnnotationResponses() + .get(1).getEntityLogEntry().getType()); + assertEquals(newLegal, + manualRedactionResponse.getManualAnnotationResponses() + .get(29).getEntityLogEntry().getLegalBasis()); + assertEquals(otherSection, + manualRedactionResponse.getManualAnnotationResponses() + .get(79).getEntityLogEntry().getSection()); Long commentId2 = manualRedactionResponse.getManualAnnotationResponses() .get(0).getCommentId(); assertNotNull(commentId2); @@ -3397,26 +3403,28 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertEquals(comment2, commentEntity2.get().getText()); String comment3 = "Recategorizing Darth Vader with type2"; - manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal( - dossier.getId(), - file.getId(), - RecategorizationBulkLocalRequestModel.builder() - .rectangle(false) - .type(newType.getType()) - .legalBasis(newLegal) - .section(otherSection) - .value(darthVader) - .originTypes(Set.of(type2.getType())) - .comment(comment3) - .build(), - false); + manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(), + file.getId(), + RecategorizationBulkLocalRequestModel.builder() + .rectangle(false) + .type(newType.getType()) + .legalBasis(newLegal) + .section(otherSection) + .value(darthVader) + .originTypes(Set.of(type2.getType())) + .comment(comment3) + .build(), + false); assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size()); - assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses() - .get(0).getEntityLogEntry().getType()); - assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses() - .get(36).getEntityLogEntry().getLegalBasis()); - assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses() - .get(98).getEntityLogEntry().getSection()); + assertEquals(newType.getType(), + manualRedactionResponse.getManualAnnotationResponses() + .get(0).getEntityLogEntry().getType()); + assertEquals(newLegal, + manualRedactionResponse.getManualAnnotationResponses() + .get(36).getEntityLogEntry().getLegalBasis()); + assertEquals(otherSection, + manualRedactionResponse.getManualAnnotationResponses() + .get(98).getEntityLogEntry().getSection()); Long commentId3 = manualRedactionResponse.getManualAnnotationResponses() .get(0).getCommentId(); assertNotNull(commentId3); @@ -3425,15 +3433,14 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertEquals(comment3, commentEntity3.get().getText()); String comment4 = "Removing all Darth Vader annotations"; - manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal( - dossier.getId(), - file.getId(), - RemoveRedactionBulkLocalRequestModel.builder() - .rectangle(false) - .value(darthVader) - .comment(comment4) - .build(), - false); + manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), + file.getId(), + RemoveRedactionBulkLocalRequestModel.builder() + .rectangle(false) + .value(darthVader) + .comment(comment4) + .build(), + false); assertEquals(202, manualRedactionResponse.getManualAnnotationResponses().size()); Long commentId4 = manualRedactionResponse.getManualAnnotationResponses() .get(0).getCommentId(); @@ -3444,7 +3451,6 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } - private void testRectangleRedactionsBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) { whenGetEntityLogInvocation(); @@ -3768,6 +3774,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { assertEquals(EntryState.APPLIED, rectangleAnnotationEntry.getState()); String awesomeComment = "awesome comment"; + String myUser = "MyUser"; BulkLocalResponse addValueBackResponse = BulkLocalResponse.builder() .fileId(file.getId()) .dossierId(dossier.getId()) @@ -3777,6 +3784,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { .section(section) .foundTerms(List.of(new FoundTerm(List.of(position), value))) .comment(awesomeComment) + .userId(myUser) .build(); searchTermOccurrencesResponseReceiver.receive(addValueBackResponse); @@ -3801,6 +3809,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { List commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), newValueAnnotation.getId(), false); assertEquals(commentEntities.size(), 1); assertEquals(commentEntities.get(0).getText(), awesomeComment); + assertEquals(commentEntities.get(0).getUser(), myUser); removeValueResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(), file.getId(), removeValueRequest, true); @@ -3843,6 +3852,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { commentEntities = commentRepository.findByFileIdAndAnnotationId(file.getId(), appliedValueAnnotations.get(0).getId(), false); assertEquals(commentEntities.size(), 1); assertEquals(commentEntities.get(0).getText(), awesomeComment); + assertEquals(commentEntities.get(0).getUser(), myUser); String totallyDifferentComment = "totally different comment"; RemoveRedactionBulkLocalRequestModel removeRectangleRequest = RemoveRedactionBulkLocalRequestModel.builder() @@ -3924,6 +3934,78 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { } + @Test + public void testLocalAddWithRectangle() { + + whenGetEntityLogInvocation(); + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + var file = fileTesterAndProvider.testAndProvideFile(dossier); + + var type = typeProvider.testAndProvideType(dossierTemplate, dossier, "test", false, 100); + + var entityLog = new EntityLog(1, 1, new ArrayList<>(), null, 0, 0, 0, 0); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + + AddRedactionBulkLocalRequestModel addRedactionRequest = AddRedactionBulkLocalRequestModel.builder() + .type(type.getType()) + .value("non-readable content") + .legalBasis("legal basis") + .reason("reason") + .section("section") + .comment("test comment") + .positions(List.of(new Rectangle(100f, 200f, 50f, 50f, 0))) + .pageNumbers(Set.of(1, 2, 3)) + .rectangle(true) + .build(); + + ManualRedactionResponse response = manualRedactionClient.addRedactionBulkLocal(dossier.getId(), file.getId(), addRedactionRequest); + + assertNotNull(response); + assertEquals(3, response.getManualAnnotationResponses().size()); + + EntityLog currentEntityLog = entityLogService.getEntityLog(dossier.getId(), file.getId()); + assertNotNull(currentEntityLog); + assertEquals(3, currentEntityLog.getEntityLogEntry().size()); + + for (int i = 1; i <= 3; i++) { + int page = i; + EntityLogEntry entry = currentEntityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getPositions() != null + && !e.getPositions().isEmpty() + && e.getPositions() + .get(0).getPageNumber() == page) + .findFirst() + .orElse(null); + assertNotNull(entry); + assertEquals("non-readable content", entry.getValue()); + assertEquals("legal basis", entry.getLegalBasis()); + assertEquals("reason", entry.getReason()); + assertEquals("section", entry.getSection()); + assertEquals(type.getType(), entry.getType()); + assertEquals(EntryType.AREA, entry.getEntryType()); + assertEquals(EntryState.APPLIED, entry.getState()); + assertEquals(1, entry.getPositions().size()); + Position position = entry.getPositions() + .get(0); + assertEquals(page, position.getPageNumber()); + assertEquals(100f, position.x()); + assertEquals(200f, position.y()); + assertEquals(50f, position.w()); + assertEquals(50f, position.h()); + } + + for (EntityLogEntry entry : currentEntityLog.getEntityLogEntry()) { + List comments = commentRepository.findByFileIdAndAnnotationId(file.getId(), entry.getId(), false); + assertEquals(1, comments.size()); + CommentEntity comment = comments.get(0); + assertEquals("test comment", comment.getText()); + } + } + + private Rectangle toRectangle(Position position) { return new Rectangle(position.x(), position.y(), position.w(), position.h(), 0); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java index 1b4e793cf..393155b13 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SearchTermOccurrencesTest.java @@ -69,6 +69,7 @@ public class SearchTermOccurrencesTest extends AbstractPersistenceServerServiceT .section("section") .foundTerms(List.of(new FoundTerm(List.of(new Position(new float[]{1f, 2f, 3f, 4f}, 1)), "searchTerm"))) .comment(null) + .userId("MyUser") .build()); List newEntries = manualRedactionRepository.findByFileIdAndOptions(file.getId(), false, false, false); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java index 59fdcd412..ecb81e1d2 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalRequest.java @@ -36,4 +36,7 @@ public class BulkLocalRequest { private String comment; + @NonNull + private String userId; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java index 3fa1a460d..a679f9ff4 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/BulkLocalResponse.java @@ -39,4 +39,6 @@ public class BulkLocalResponse { private String comment; + @NonNull + private String userId; } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java index 7b4d8a0d0..7ba094484 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/manual/AddRedactionBulkLocalRequestModel.java @@ -35,6 +35,8 @@ public class AddRedactionBulkLocalRequestModel { private String section; + private boolean rectangle; + @NonNull @Builder.Default private List positions = new ArrayList<>(); From c52dfbdf5abb056940172c4dab12408c546bda2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Mon, 14 Oct 2024 15:30:18 +0200 Subject: [PATCH 4/4] RED-10127: Import/Export of dossiertemplate with preview files --- .../impl/controller/SupportController.java | 22 +++ .../external/resource/SupportResource.java | 23 +++ .../models/DatasetExchangeImportModel.java | 57 ++++++ .../service/DatasetExchangeService.java | 183 ++++++++++++++++++ .../service/DossierImportService.java | 2 +- .../service/DossierTemplateExportService.java | 4 +- .../DatasetExchangeArchiveReader.java | 50 +++++ .../zipreaders/ZipEntryIterator.java | 8 +- .../download/DownloadPreparationService.java | 26 ++- .../DownloadReportCleanupService.java | 6 +- .../DossierPersistenceService.java | 9 +- .../DownloadStatusPersistenceService.java | 4 +- .../repository/DossierRepository.java | 5 + .../utils/FileSystemBackedArchiver.java | 3 +- .../build.gradle.kts | 4 +- .../dossiertemplate/DownloadFileType.java | 3 +- 16 files changed, 392 insertions(+), 17 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index 66a222fcd..e0b802fce 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService; @@ -48,6 +49,7 @@ public class SupportController implements SupportResource { private final FileStatusManagementService fileStatusManagementService; private final FileExchangeExportService fileExchangeExportService; private final FileExchangeImportService fileExchangeImportService; + private final DatasetExchangeService datasetExchangeService; @Override @@ -127,6 +129,13 @@ public class SupportController implements SupportResource { } + @Override + public DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId) { + + return datasetExchangeService.prepareExport(dossierTemplateId); + } + + @Override public DownloadResponse exportFiles(String dossierTemplateId, FileExchangeExportRequest exportRequest) { @@ -147,4 +156,17 @@ public class SupportController implements SupportResource { return fileExchangeImportService.importFileExchangeArchive(KeycloakSecurity.getUserId(), bytes); } + @Override + @PreAuthorize("hasAuthority('" + IMPORT_FILES + "')") + public ImportResponse importDataset(MultipartFile file) { + + byte[] bytes; + try { + bytes = file.getBytes(); + } catch (IOException e) { + throw new BadRequestException("File could not be read and is likely corrupted.", e); + } + return datasetExchangeService.importDataset(KeycloakSecurity.getUserId(), bytes); + } + } diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index b8723183d..5782e030f 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -34,6 +34,7 @@ public interface SupportResource { String STATUS_REST_PATH = ExternalApi.BASE_PATH + "/status/filter"; String FILE_EXCHANGE_REST_PATH = ExternalApi.BASE_PATH + "/file-exchange"; + String DATASET_EXCHANGE = ExternalApi.BASE_PATH + "/dataset-exchange"; String BULK_REST_PATH = "/bulk"; @@ -143,6 +144,21 @@ public interface SupportResource { DownloadResponse exportFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody FileExchangeExportRequest exportRequest); + @ResponseStatus(value = HttpStatus.OK) + @ResponseBody + @PostMapping(value = DATASET_EXCHANGE + + EXPORT + + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Exports all dossiers and files from a given Dossier Template.", description = """ + ## Export Preview Files Endpoint + + Use this endpoint to export all files of a DossierTemplate as a Preview file containing all applied annotations as redaction annotations with additional data + The endpoint returns a String storageId, which is used to query the DownloadController for the export zip archive's status and to download the archive. + """) + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) + DownloadResponse exportDataset(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId); + + @ResponseBody @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) @@ -150,4 +166,11 @@ public interface SupportResource { @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) ImportResponse importFiles(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file); + @ResponseBody + @ResponseStatus(value = HttpStatus.OK) + @PostMapping(value = DATASET_EXCHANGE + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. Returns the resulting dossierTemplateId.") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) + ImportResponse importDataset(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java new file mode 100644 index 000000000..a447758df --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/DatasetExchangeImportModel.java @@ -0,0 +1,57 @@ +package com.iqser.red.service.persistence.management.v1.processor.dataexchange.models; + +import java.util.LinkedList; +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DatasetExchangeImportModel { + + private ImportTemplateResult importTemplateResult = ImportTemplateResult.builder().build(); + private List dossiers = new LinkedList<>(); + + @Builder + @Getter + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) + public static class Dossier { + + String name; + @Builder.Default + List files = new LinkedList<>(); + + } + + @Getter + @AllArgsConstructor + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) + public static class File { + + String name; + byte[] data; + + } + + + public DatasetExchangeImportModel.Dossier getOrCreateDossier(String name) { + + for (Dossier dossier : dossiers) { + if (dossier.getName().equals(name)) { + return dossier; + } + } + Dossier dossier = Dossier.builder().name(name).build(); + dossiers.add(dossier); + return dossier; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java new file mode 100644 index 000000000..ab765e3c6 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DatasetExchangeService.java @@ -0,0 +1,183 @@ +package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service; + +import static com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeArchivalService.SIZE_THRESHOLD; + +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; +import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.TemplateImportInfo; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.DatasetExchangeArchiveReader; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders.ZipEntryIterator; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; +import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.UploadService; +import com.iqser.red.service.persistence.management.v1.processor.service.download.DownloadPreparationService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ImportResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue; +import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DatasetExchangeService { + + DossierTemplateManagementService dossierTemplateManagementService; + DownloadStatusPersistenceService downloadStatusPersistenceService; + FileStatusPersistenceService fileStatusPersistenceService; + DossierPersistenceService dossierPersistenceService; + ColorsService colorsService; + RabbitTemplate rabbitTemplate; + FileManagementServiceSettings settings; + DossierTemplateImportService dossierTemplateImportService; + DossierCreatorService dossierCreatorService; + ReportTemplatePersistenceService reportTemplateService; + UploadService uploadService; + DownloadPreparationService downloadPreparationService; + FileManagementStorageService storageService; + + + public DownloadResponse prepareExport(String dossierTemplateId) { + + var template = dossierTemplateManagementService.getDossierTemplate(dossierTemplateId); + String downloadFilename = template.getName() + "_PREVIEW_FILES.zip"; + String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_PREVIEW_FILES"); + List files = fileStatusPersistenceService.getStatusesForDossierTemplate(dossierTemplateId); + List dossiers = dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); + if (dossiers.isEmpty()) { + throw new BadRequestException("No dossiers in DossierTemplate with id" + dossierTemplateId); + } + ColorsEntity colors = colorsService.getColors(dossierTemplateId); + + downloadPreparationService.clearRedactionStatusEntries(storageId); + if (storageService.objectExists(storageId)) { + storageService.deleteObject(storageId); + } + + DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.createStatus(KeycloakSecurity.getUserId(), + storageId, + dossiers.get(0), + downloadFilename, + "application/zip", + files.stream() + .map(FileEntity::getId) + .toList(), + Set.of(DownloadFileType.DATASET_EXPORT), + Collections.emptyList(), + colors.getPreviewColor()); + + RedactionMessage.RedactionMessageBuilder builder = RedactionMessage.builder() + .appliedRedactionColor(colors.getRedactionColor()) + .redactionPreviewColor(colors.getPreviewColor()) + .keepHiddenText(true) + .keepOverlappingObjects(true) + .keepImageMetaData(true) + .downloadId(storageId) + .redactionTypes(List.of(RedactionType.PREVIEW)); + + downloadStatus.getFiles() + .forEach(fileEntity -> { + RedactionMessage message = builder.dossierId(fileEntity.getDossierId()) + .fileId(fileEntity.getId()) + .unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED) + .build(); + log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", storageId, fileEntity.getId()); + rabbitTemplate.convertAndSend(MessagingConfiguration.PDFTRON_REQUEST_EXCHANGE, TenantContext.getTenantId(), message); + }); + + downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.GENERATING); + + return new DownloadResponse(storageId); + } + + + @SneakyThrows + public ImportResponse importDataset(String userId, byte[] archive) { + + DatasetExchangeArchiveReader datasetExchangeArchiveReader = new DatasetExchangeArchiveReader(userId); + try (ZipEntryIterator zipEntryIterator = new ZipEntryIterator(archive, settings.getCompressionThresholdRatio(), SIZE_THRESHOLD)) { + zipEntryIterator.forEachRemaining(datasetExchangeArchiveReader::handleZipEntryData); + } + DatasetExchangeImportModel importModel = datasetExchangeArchiveReader.getDatasetExchangeImportModel(); + + TemplateImportInfo templateImportInfo = dossierTemplateImportService.importDossierTemplate(importModel.getImportTemplateResult()); + for (DatasetExchangeImportModel.Dossier dossier : importModel.getDossiers()) { + + String dossierId = getDossierForImport(templateImportInfo, dossier, userId); + for (DatasetExchangeImportModel.File file : dossier.getFiles()) { + uploadService.processSingleFile(dossierId, file.getName(), file.getData(), false, false); + } + } + return new ImportResponse(templateImportInfo.getDossierTemplateId()); + } + + + private String getDossierForImport(TemplateImportInfo templateImportInfo, DatasetExchangeImportModel.Dossier dossier, String userId) { + + String dossierTemplateId = templateImportInfo.getDossierTemplateId(); + + CreateOrUpdateDossierRequest request = CreateOrUpdateDossierRequest.builder() + .dossierName(dossier.getName()) + .description(dossier.getName()) + .downloadFileTypes(Collections.emptySet()) + .dossierTemplateId(dossierTemplateId) + .requestingUser(userId) + .reportTemplateIds(reportTemplateService.findByDossierTemplateId(templateImportInfo.getDossierTemplateId()) + .stream() + .map(ReportTemplateEntity::getTemplateId) + .toList()) + .build(); + + Dossier importedDossier = null; + int retries = 0; + while (importedDossier == null && retries < 100) { + try { + importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId); + } catch (ConflictException e) { + retries++; + request.setDossierName(String.format("%s (%d)", dossier.getName(), retries)); + } + } + if (importedDossier == null) { + throw new BadRequestException(String.format("Could not create dossier with name %s in %d retries", dossier.getName(), retries)); + } + return importedDossier.getId(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java index f309674e3..bf1ce59e0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierImportService.java @@ -67,7 +67,7 @@ public class DossierImportService { Dossier importedDossier = null; int retries = 0; - while (importedDossier == null && retries < 10) { + while (importedDossier == null && retries < 100) { try { importedDossier = dossierCreatorService.addDossier(request, Set.of(userId), Set.of(userId), userId); } catch (ConflictException e) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java index 61a56b934..03d2b8ef7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierTemplateExportService.java @@ -72,6 +72,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @@ -150,8 +151,9 @@ public class DossierTemplateExportService { } + @SneakyThrows @Observed(name = "DossierTemplateExportService", contextualName = "write-dossier-template-to-archive") - public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) throws IOException { + public void addDossierTemplateToArchive(FileSystemBackedArchiver fileSystemBackedArchiver, String folder, DossierTemplateEntity dossierTemplate) { // add dossier template name and meta data fileSystemBackedArchiver.addEntries(new FileSystemBackedArchiver.ArchiveModel(folder, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java new file mode 100644 index 000000000..97291063a --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/DatasetExchangeArchiveReader.java @@ -0,0 +1,50 @@ +package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders; + +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.DatasetExchangeImportModel; + +import lombok.AccessLevel; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; + +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DatasetExchangeArchiveReader { + + DossierTemplateArchiveReader dossierTemplateArchiveReader; + DatasetExchangeImportModel datasetExchangeImportModel; + + + public DatasetExchangeArchiveReader(String userId) { + + dossierTemplateArchiveReader = new DossierTemplateArchiveReader(userId, null, false); + datasetExchangeImportModel = new DatasetExchangeImportModel(); + } + + + public DatasetExchangeImportModel getDatasetExchangeImportModel() { + + datasetExchangeImportModel.setImportTemplateResult(dossierTemplateArchiveReader.buildResult()); + return datasetExchangeImportModel; + } + + + @SneakyThrows + public void handleZipEntryData(ZipEntryData zipEntryData) { + + if (zipEntryData.getBaseFolder().equals(FileExchangeNames.TEMPLATE_FOLDER)) { + dossierTemplateArchiveReader.handleZipEntryData(zipEntryData); + } else if (zipEntryData.getDepth() >= 1) { // in dossier folders + + String dossierName = zipEntryData.getBaseFolder().replaceAll("\\s*\\([1-9]+\\)", ""); + String fileName = zipEntryData.getFileName(); + if (!fileName.endsWith(".pdf")) { + return; + } + + var dossier = datasetExchangeImportModel.getOrCreateDossier(dossierName); + dossier.getFiles().add(new DatasetExchangeImportModel.File(fileName, zipEntryData.data())); + } + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java index 2669501dd..506b09209 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/ZipEntryIterator.java @@ -1,6 +1,10 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.zipreaders; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.UUID; @@ -70,7 +74,7 @@ public class ZipEntryIterator implements Iterator, AutoCloseable { double compressionRatio = (double) totalSizeEntry / nextEntry.getCompressedSize(); if (compressionRatio > compressionThresholdRatio) { - throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio ); + throw new BadRequestException("ZIP-Bomb detected (compressionRatio). " + compressionRatio + "/" + compressionThresholdRatio); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java index 6e6b2050f..f45a7c38b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.download; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -21,6 +22,8 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetai import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultMessage; import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionType; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; +import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; @@ -29,12 +32,13 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.download import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadRedactionFileStatusRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.websocket.WebsocketService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest; @@ -68,6 +72,8 @@ public class DownloadPreparationService { private final DossierTemplatePersistenceService dossierTemplatePersistenceService; private final DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository; private final WebsocketService websocketService; + private final DossierPersistenceService dossierPersistenceService; + private final DossierTemplateExportService dossierTemplateExportService; @Value("${storage.backend}") private String storageBackend; @@ -134,7 +140,7 @@ public class DownloadPreparationService { for (var downloadFileType : downloadFileTypes) { switch (downloadFileType) { case REDACTED -> result.add(RedactionType.REDACTED); - case PREVIEW -> result.add(RedactionType.PREVIEW); + case PREVIEW, DATASET_EXPORT -> result.add(RedactionType.PREVIEW); case OPTIMIZED_PREVIEW -> result.add(RedactionType.OPTIMIZED_PREVIEW); case DELTA_PREVIEW -> result.add(RedactionType.DELTA); default -> { @@ -198,7 +204,7 @@ public class DownloadPreparationService { var redactionFileResults = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(downloadId); DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadId); - var storedFileInformations = getStoredFileInformation(downloadId); + List storedFileInformations = getStoredFileInformation(downloadId); try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) { @@ -276,6 +282,11 @@ public class DownloadPreparationService { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(file.getFilename(), "redacted"), // getRedacted(file.getId(), redactionFileResult.get().getDetails()))); } + if (downloadFileType.equals(DownloadFileType.DATASET_EXPORT)) { + String dossierName = dossierPersistenceService.getDossierNameById(file.getDossierId()); + fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierName, file.getFilename(),// + getPreview(file.getId(), redactionFileResult.get().getDetails()))); + } } log.info("Successfully added file {}/{} for downloadId {}, took {}", @@ -285,6 +296,10 @@ public class DownloadPreparationService { System.currentTimeMillis() - start); i++; } + if (downloadStatus.getDownloadFileTypes().contains(DownloadFileType.DATASET_EXPORT)) { + DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadStatus.getDossier().getDossierTemplateId()); + dossierTemplateExportService.addDossierTemplateToArchive(fileSystemBackedArchiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate); + } log.info("Successfully added {} files for downloadId {}, took {}", downloadStatus.getFiles() .stream() @@ -381,7 +396,10 @@ public class DownloadPreparationService { } else { storageId = generateReportJsonStorageIdForAzure(downloadId); } - + if (!fileManagementStorageService.objectExists(storageId)) { + log.warn("Could not find stored file information with id {}", storageId); + return new ArrayList<>(); + } return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() { }); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java index c2a69b616..3c9b9ebb9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadReportCleanupService.java @@ -22,8 +22,10 @@ public class DownloadReportCleanupService { public void deleteTmpReportFiles(Collection storageIds) { for (String storageId : storageIds) { - fileManagementStorageService.deleteObject(storageId); - log.info("Deleted tmp report file {}", storageId); + if (fileManagementStorageService.objectExists(storageId)) { + fileManagementStorageService.deleteObject(storageId); + log.info("Deleted tmp report file {}", storageId); + } } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 21a03fb26..6f8f21c69 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -209,7 +209,7 @@ public class DossierPersistenceService { @Transactional - public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) { + public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) { dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } @@ -275,4 +275,11 @@ public class DossierPersistenceService { return dossierRepository.findDossierTemplateId(dossierId); } + + public String getDossierNameById(String dossierId) { + + return dossierRepository.findDossierNameById(dossierId) + .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE)); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java index a9ec3de3c..6dea74c67 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java @@ -38,7 +38,7 @@ public class DownloadStatusPersistenceService { } - public void createStatus(String userId, + public DownloadStatusEntity createStatus(String userId, String storageId, DossierEntity dossier, String filename, @@ -62,7 +62,7 @@ public class DownloadStatusPersistenceService { downloadStatus.setRedactionPreviewColor(redactionPreviewColor); downloadStatus.setStatus(DownloadStatusValue.QUEUED); - downloadStatusRepository.save(downloadStatus); + return downloadStatusRepository.save(downloadStatus); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 438abb196..a844fc87a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.util.List; +import java.util.Optional; import java.util.Set; import org.springframework.data.jpa.repository.JpaRepository; @@ -110,4 +111,8 @@ public interface DossierRepository extends JpaRepository @Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold") List findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold); + + @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL") + Optional findDossierNameById(@Param("dossierId") String dossierId); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java index 1bb88f481..d2f354de2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileSystemBackedArchiver.java @@ -6,6 +6,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; @@ -50,7 +51,7 @@ public class FileSystemBackedArchiver implements AutoCloseable { this.rethrowExceptions = rethrowExceptions; tempFile = FileUtils.createTempFile("archive", ".zip"); - zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile)); + zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile), StandardCharsets.UTF_8); } diff --git a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts index 3a667d791..f03ed5f7a 100644 --- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts @@ -41,19 +41,19 @@ description = "persistence-service-server-v1" tasks.named("bootBuildImage") { - builder.set("docker-proxy.knecon.com/paketobuildpacks/builder:base") runImage.set("docker-proxy.knecon.com/paketobuildpacks/run:base-cnb") environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ") environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8") + environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // FileInputStream or ZipInputStream does not seem to care for file.encoding imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}") if (project.hasProperty("buildbootDockerHostNetwork")) { network.set("host") } - docker { + docker { if (project.hasProperty("buildbootDockerHostNetwork")) { bindHostToBuilder.set(true) } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java index f90cb5e37..e8cd137d6 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/DownloadFileType.java @@ -7,5 +7,6 @@ public enum DownloadFileType { REDACTED, ANNOTATED, FLATTEN, - DELTA_PREVIEW + DELTA_PREVIEW, + DATASET_EXPORT }