Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9782: Enabled to upload file and import only imported redactions with... #640

Merged
dominique.eiflaender1 merged 1 commits from RED-9782-4.1 into release/2.465.x 2024-08-02 10:39:38 +02:00
Conversation 7 Commits 1 Files Changed 12 +44 -28
12 changed files with 44 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java vendored
View File

@ -41,6 +41,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
import feign.FeignException;
import io.micrometer.core.annotation.Timed;
import io.swagger.v3.oas.annotations.Parameter;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@ -67,7 +68,8 @@ public class UploadController implements UploadResource {
@Override
public FileUploadResult upload(@RequestPart(name = "file") MultipartFile file,
@PathVariable(DOSSIER_ID) String dossierId,
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions) {
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction) {
accessControlService.checkAccessPermissionsToDossier(dossierId);
if (file.getOriginalFilename() == null) {
@ -79,7 +81,7 @@ public class UploadController implements UploadResource {
try {
switch (extension) {
case "zip":
return handleZip(dossierId, file.getBytes(), keepManualRedactions);
return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticRedaction);
case "csv":
return uploadService.importCsv(dossierId, file.getBytes());
default:
@ -89,7 +91,7 @@ public class UploadController implements UploadResource {
if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
throw new NotAllowedException("Insufficient permissions");
}
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions);
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticRedaction);
}
} catch (IOException e) {
throw new BadRequestException(e.getMessage(), e);
@ -133,7 +135,7 @@ public class UploadController implements UploadResource {
}
private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions) throws IOException {
private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticRedaction) throws IOException {
File tempFile = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
@ -143,7 +145,7 @@ public class UploadController implements UploadResource {
try {
checkForSymlinks(tempFile);
var zipData = unzip(tempFile, dossierId, keepManualRedactions);
var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticRedaction);
if (zipData.csvBytes != null) {
try {
@ -186,7 +188,7 @@ public class UploadController implements UploadResource {
}
private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions) throws IOException {
private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticRedaction) throws IOException {
var zipData = new ZipData();
@ -197,7 +199,7 @@ public class UploadController implements UploadResource {
zipData.totalEntryArchive++;
if (!ze.isDirectory()) {
processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData);
processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticRedaction);
}
}
}
@ -205,7 +207,7 @@ public class UploadController implements UploadResource {
}
private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData) throws IOException {
private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticRedaction) throws IOException {
var extension = getExtension(ze.getName());
@ -240,7 +242,7 @@ public class UploadController implements UploadResource {
zipData.containedUnpermittedFiles = false;
try {
var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions);
var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticRedaction);
zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
} catch (Exception e) {
log.debug("PDF File inside ZIP failed", e);

6
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java vendored
View File

@ -38,6 +38,7 @@ import com.iqser.red.service.persistence.service.v2.api.external.model.FileDelet
import com.iqser.red.service.persistence.service.v2.api.external.model.FileStatusList;
import com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
@ -60,11 +61,12 @@ public class FileControllerV2 implements FileResource {
public FileUploadResult upload(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestPart(name = FILE_PARAM) MultipartFile file,
@RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges) {
@RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges,
@RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction) {
dossierTemplateController.getDossierTemplate(dossierTemplateId);
return uploadController.upload(file, dossierId, keepManualChanges);
return uploadController.upload(file, dossierId, keepManualChanges, disableAutomaticRedaction);
}

5
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/UploadResource.java vendored
View File

@ -15,6 +15,7 @@ import org.springframework.web.multipart.MultipartFile;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
@ -27,6 +28,7 @@ public interface UploadResource {
String IMPORT_REDACTIONS_PATH = ExternalApi.BASE_PATH + "/import-redactions";
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
String DISABLE_AUTOMATIC_REDACTION_PARAM = "disableAutomaticRedaction";
@ResponseBody
@ -37,7 +39,8 @@ public interface UploadResource {
+ "uploaded file."), @ApiResponse(responseCode = "404", description = "Dossier not found"), @ApiResponse(responseCode = "403", description = "Forbidden")})
FileUploadResult upload(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@PathVariable(DOSSIER_ID) String dossierId,
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions);
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction);
@ResponseBody

4
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/FileResource.java vendored
View File

@ -48,6 +48,7 @@ public interface FileResource {
String CREATE_DOWNLOAD_PATH = "/create-download";
String BULK_CREATE_DOWNLOAD_PATH = "/bulk/create-download";
String KEEP_MANUAL_CHANGES_PARAM = "keepManualChanges";
String DISABLE_AUTOMATIC_REDACTION_PARAM = "disableAutomaticRedaction";
String DELETE_PERMANENTLY_PARAM = "deletePermanently";
String FILE_PARAM = "file";
String FILE_ID_PARAM = "fileId";
@ -62,7 +63,8 @@ public interface FileResource {
FileUploadResult upload(@Parameter(name = DOSSIER_TEMPLATE_ID_PARAM, description = "The identifier of the dossier template that is used for the dossier.", required = true) @PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId,
@Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier to where the file is to be uploaded.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
@Parameter(name = FILE_PARAM, description = "The file to be uploaded.", required = true) @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = FILE_PARAM) MultipartFile file,
@Parameter(name = KEEP_MANUAL_CHANGES_PARAM, description = "A Toggle to keep manual changes: Manual changes are manually added annotations or manipulations on existing ones. If set to `true` the system keeps the manual changes on re-uploading (overwriting) the file.") @RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges);
@Parameter(name = KEEP_MANUAL_CHANGES_PARAM, description = "A Toggle to keep manual changes: Manual changes are manually added annotations or manipulations on existing ones. If set to `true` the system keeps the manual changes on re-uploading (overwriting) the file.") @RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction);
@ResponseStatus(value = HttpStatus.OK)

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java
View File

@ -52,7 +52,7 @@ public class FileService {
private final WebsocketService websocketService;
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size) {
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size, boolean disableAutomaticRedaction) {
dossierPersistenceService.getAndValidateDossier(request.getDossierId());
@ -75,7 +75,7 @@ public class FileService {
} else {
// the file is new, should create a new status for it.
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), size);
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), size, disableAutomaticRedaction);
}
return new JSONPrimitive<>(request.getFileId());

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -285,6 +285,11 @@ public class FileStatusService {
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.build();
if(fileEntity.isExcluded() && fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)){
analyseRequest.setMessageType(MessageType.IMPORTED_REDACTIONS_ONLY);
}
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
if (messageType.equals(MessageType.REANALYSE)) {
@ -612,9 +617,9 @@ public class FileStatusService {
@Transactional
public void createStatus(String dossierId, String fileId, String uploader, String filename, long size) {
public void createStatus(String dossierId, String fileId, String uploader, String filename, long size, boolean disableAutomaticRedaction) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size);
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size, disableAutomaticRedaction);
websocketService.sendFileEvent(dossierId, fileId, FileEventType.CREATE);
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java
View File

@ -64,7 +64,7 @@ public class UploadService {
@PreAuthorize("hasAuthority('" + UPLOAD_FILE + "')")
public FileUploadResult processSingleFile(String dossierId, String fileName, byte[] fileContent, boolean keepManualRedactions) {
public FileUploadResult processSingleFile(String dossierId, String fileName, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticRedaction) {
dossierManagementService.getDossierById(dossierId, false, false);
@ -76,7 +76,7 @@ public class UploadService {
try {
storageService.storeObject(TenantContext.getTenantId(), storageId, new ByteArrayInputStream(fileContent));
fileService.upload(new AddFileRequest(fileName, fileId, dossierId, KeycloakSecurity.getUserId()), keepManualRedactions, fileContent.length);
fileService.upload(new AddFileRequest(fileName, fileId, dossierId, KeycloakSecurity.getUserId()), keepManualRedactions, fileContent.length, disableAutomaticRedaction);
} catch (Exception e) {
storageService.deleteObject(TenantContext.getTenantId(), storageId);

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -50,7 +50,7 @@ public class FileStatusPersistenceService {
private final EntityManager entityManager;
public FileEntity createStatus(String dossierId, String fileId, String filename, String uploader, long size) {
public FileEntity createStatus(String dossierId, String fileId, String filename, String uploader, long size, boolean disableAutomaticRedaction) {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
FileEntity file = new FileEntity();
@ -68,6 +68,7 @@ public class FileStatusPersistenceService {
file.setProcessingErrorCounter(0);
file.setFileSize(size);
file.setComponentMappingVersions(new ArrayList<>());
file.setExcludedFromAutomaticAnalysis(disableAutomaticRedaction);
return fileRepository.save(file);
}

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -96,7 +96,7 @@ public class FileTesterAndProvider {
AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1");
fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
var mff = new MockMultipartFile(fileName + ".pdf", fileName + ".pdf", "application/pdf", "lorem ipsum".getBytes());
var uploadResult = uploadClient.upload(mff, dossier.getId(), false);
var uploadResult = uploadClient.upload(mff, dossier.getId(), false, false);
return uploadResult.getFileIds()
.iterator().next();

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileAttributeTest.java
View File

@ -161,7 +161,7 @@ public class FileAttributeTest extends AbstractPersistenceServerServiceTest {
.iterator().next().getValue()).isNull();
var mock = new MockMultipartFile("test.csv", "test.csv", "application/csv", IOUtils.toByteArray(new ClassPathResource("files/test.csv").getInputStream()));
uploadClient.upload(mock, dossier.getId(), false);
uploadClient.upload(mock, dossier.getId(), false, false);
fileAttributes = fileClient.getFileStatus(dossier.getId(), file.getId()).getFileAttributes().getAttributeIdToValue();
assertThat(fileAttributes.size()).isEqualTo(2);

10
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -191,7 +191,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(viewedPages.getPages().size()).isEqualTo(1);
var fileUpload = new MockMultipartFile("test.pdf", "test.pdf", "application/pdf", "content".getBytes());
var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false);
var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false, false);
loadedFile = fileClient.getFileStatus(dossier.getId(),
uploadResult.getFileIds()
@ -232,7 +232,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(viewedPages.getPages()).hasSize(1);
var fileUpload = new MockMultipartFile(file.getFilename(), file.getFilename(), "application/pdf", "content".getBytes());
var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false);
var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false, false);
loadedFile = fileClient.getFileStatus(dossier.getId(),
uploadResult.getFileIds()
@ -686,7 +686,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.ORIGIN, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
MockMultipartFile mff = new MockMultipartFile(fileName, fileName + ".pdf", "application/pdf", "lorem ipsum".getBytes());
FileUploadResult uploadResult = uploadClient.upload(mff, dossier.getId(), false);
FileUploadResult uploadResult = uploadClient.upload(mff, dossier.getId(), false, false);
fileId = uploadResult.getFileIds()
.iterator().next();
@ -725,7 +725,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
.encoding("UTF-8")
.build());
when(fileAttributeConfigPersistenceService.getFileAttributes(anyString())).thenReturn(Collections.emptyList());
assertThrows(FeignException.class, () -> uploadClient.upload(malformedCsvFile, dossier.getId(), false));
assertThrows(FeignException.class, () -> uploadClient.upload(malformedCsvFile, dossier.getId(), false, false));
}
@ -795,7 +795,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
//overwrite the file
var fileUpload = new MockMultipartFile(file.getFilename(), file.getFilename(), "application/pdf", "content".getBytes());
var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false);
var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false, false);
loadedFile = fileClient.getFileStatus(dossier.getId(),
uploadResult.getFileIds()

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/MessageType.java
View File

@ -4,5 +4,6 @@ public enum MessageType {
ANALYSE,
REANALYSE,
SURROUNDING_TEXT_ANALYSIS
SURROUNDING_TEXT_ANALYSIS,
IMPORTED_REDACTIONS_ONLY
}