Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9782: Fixed bugs in upload with disabledAutomaticAnalysis #643

Merged
dominique.eiflaender1 merged 1 commits from RED-9782-4.1 into release/2.465.x 2024-08-02 14:41:44 +02:00
Conversation 5 Commits 1 Files Changed 8 +24 -25
8 changed files with 24 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java vendored
View File

@ -69,7 +69,7 @@ public class UploadController implements UploadResource {
public FileUploadResult upload(@RequestPart(name = "file") MultipartFile file,
@PathVariable(DOSSIER_ID) String dossierId,
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction) {
@Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) {
accessControlService.checkAccessPermissionsToDossier(dossierId);
if (file.getOriginalFilename() == null) {
@ -81,7 +81,7 @@ public class UploadController implements UploadResource {
try {
switch (extension) {
case "zip":
return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticRedaction);
return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
case "csv":
return uploadService.importCsv(dossierId, file.getBytes());
default:
@ -91,7 +91,7 @@ public class UploadController implements UploadResource {
if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
throw new NotAllowedException("Insufficient permissions");
}
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticRedaction);
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
}
} catch (IOException e) {
throw new BadRequestException(e.getMessage(), e);
@ -135,7 +135,7 @@ public class UploadController implements UploadResource {
}
private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticRedaction) throws IOException {
private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
File tempFile = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
@ -145,7 +145,7 @@ public class UploadController implements UploadResource {
try {
checkForSymlinks(tempFile);
var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticRedaction);
var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticAnalysis);
if (zipData.csvBytes != null) {
try {
@ -188,7 +188,7 @@ public class UploadController implements UploadResource {
}
private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticRedaction) throws IOException {
private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
var zipData = new ZipData();
@ -199,7 +199,7 @@ public class UploadController implements UploadResource {
zipData.totalEntryArchive++;
if (!ze.isDirectory()) {
processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticRedaction);
processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticAnalysis);
}
}
}
@ -207,7 +207,7 @@ public class UploadController implements UploadResource {
}
private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticRedaction) throws IOException {
private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticAnalysis) throws IOException {
var extension = getExtension(ze.getName());
@ -242,7 +242,7 @@ public class UploadController implements UploadResource {
zipData.containedUnpermittedFiles = false;
try {
var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticRedaction);
var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticAnalysis);
zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
} catch (Exception e) {
log.debug("PDF File inside ZIP failed", e);

5
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java vendored
View File

@ -38,7 +38,6 @@ import com.iqser.red.service.persistence.service.v2.api.external.model.FileDelet
import com.iqser.red.service.persistence.service.v2.api.external.model.FileStatusList;
import com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
@ -62,11 +61,11 @@ public class FileControllerV2 implements FileResource {
@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestPart(name = FILE_PARAM) MultipartFile file,
@RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges,
@RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction) {
@RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) {
dossierTemplateController.getDossierTemplate(dossierTemplateId);
return uploadController.upload(file, dossierId, keepManualChanges, disableAutomaticRedaction);
return uploadController.upload(file, dossierId, keepManualChanges, disableAutomaticAnalysis);
}

4
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/UploadResource.java vendored
View File

@ -28,7 +28,7 @@ public interface UploadResource {
String IMPORT_REDACTIONS_PATH = ExternalApi.BASE_PATH + "/import-redactions";
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
String DISABLE_AUTOMATIC_REDACTION_PARAM = "disableAutomaticRedaction";
String DISABLE_AUTOMATIC_ANALYSIS_PARAM = "disableAutomaticAnalysis";
@ResponseBody
@ -40,7 +40,7 @@ public interface UploadResource {
FileUploadResult upload(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@PathVariable(DOSSIER_ID) String dossierId,
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction);
@Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic analysis for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis);
@ResponseBody

4
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/FileResource.java vendored
View File

@ -48,7 +48,7 @@ public interface FileResource {
String CREATE_DOWNLOAD_PATH = "/create-download";
String BULK_CREATE_DOWNLOAD_PATH = "/bulk/create-download";
String KEEP_MANUAL_CHANGES_PARAM = "keepManualChanges";
String DISABLE_AUTOMATIC_REDACTION_PARAM = "disableAutomaticRedaction";
String DISABLE_AUTOMATIC_ANALYSIS_PARAM = "disableAutomaticAnalysis";
String DELETE_PERMANENTLY_PARAM = "deletePermanently";
String FILE_PARAM = "file";
String FILE_ID_PARAM = "fileId";
@ -64,7 +64,7 @@ public interface FileResource {
@Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier to where the file is to be uploaded.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
@Parameter(name = FILE_PARAM, description = "The file to be uploaded.", required = true) @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = FILE_PARAM) MultipartFile file,
@Parameter(name = KEEP_MANUAL_CHANGES_PARAM, description = "A Toggle to keep manual changes: Manual changes are manually added annotations or manipulations on existing ones. If set to `true` the system keeps the manual changes on re-uploading (overwriting) the file.") @RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction);
@Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis);
@ResponseStatus(value = HttpStatus.OK)

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java
View File

@ -52,7 +52,7 @@ public class FileService {
private final WebsocketService websocketService;
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size, boolean disableAutomaticRedaction) {
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size, boolean disableAutomaticAnalysis) {
dossierPersistenceService.getAndValidateDossier(request.getDossierId());
@ -75,7 +75,7 @@ public class FileService {
} else {
// the file is new, should create a new status for it.
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), size, disableAutomaticRedaction);
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), size, disableAutomaticAnalysis);
}
return new JSONPrimitive<>(request.getFileId());

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -286,7 +286,7 @@ public class FileStatusService {
.build();
if(fileEntity.isExcluded() && fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)){
if(fileEntity.isExcludedFromAutomaticAnalysis() && fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)){
analyseRequest.setMessageType(MessageType.IMPORTED_REDACTIONS_ONLY);
}
@ -617,9 +617,9 @@ public class FileStatusService {
@Transactional
public void createStatus(String dossierId, String fileId, String uploader, String filename, long size, boolean disableAutomaticRedaction) {
public void createStatus(String dossierId, String fileId, String uploader, String filename, long size, boolean disableAutomaticAnalysis) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size, disableAutomaticRedaction);
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size, disableAutomaticAnalysis);
websocketService.sendFileEvent(dossierId, fileId, FileEventType.CREATE);
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java
View File

@ -64,7 +64,7 @@ public class UploadService {
@PreAuthorize("hasAuthority('" + UPLOAD_FILE + "')")
public FileUploadResult processSingleFile(String dossierId, String fileName, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticRedaction) {
public FileUploadResult processSingleFile(String dossierId, String fileName, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) {
dossierManagementService.getDossierById(dossierId, false, false);
@ -76,7 +76,7 @@ public class UploadService {
try {
storageService.storeObject(TenantContext.getTenantId(), storageId, new ByteArrayInputStream(fileContent));
fileService.upload(new AddFileRequest(fileName, fileId, dossierId, KeycloakSecurity.getUserId()), keepManualRedactions, fileContent.length, disableAutomaticRedaction);
fileService.upload(new AddFileRequest(fileName, fileId, dossierId, KeycloakSecurity.getUserId()), keepManualRedactions, fileContent.length, disableAutomaticAnalysis);
} catch (Exception e) {
storageService.deleteObject(TenantContext.getTenantId(), storageId);

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -50,7 +50,7 @@ public class FileStatusPersistenceService {
private final EntityManager entityManager;
public FileEntity createStatus(String dossierId, String fileId, String filename, String uploader, long size, boolean disableAutomaticRedaction) {
public FileEntity createStatus(String dossierId, String fileId, String filename, String uploader, long size, boolean disableAutomaticAnalysis) {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
FileEntity file = new FileEntity();
@ -68,7 +68,7 @@ public class FileStatusPersistenceService {
file.setProcessingErrorCounter(0);
file.setFileSize(size);
file.setComponentMappingVersions(new ArrayList<>());
file.setExcludedFromAutomaticAnalysis(disableAutomaticRedaction);
file.setExcludedFromAutomaticAnalysis(disableAutomaticAnalysis);
return fileRepository.save(file);
}