Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9782: Fixed bugs in upload with disabledAutomaticAnalysis

Browse Source
This commit is contained in:
Dominique Eifländer 2024-08-02 14:25:39 +02:00
parent b1fb547b0e
commit b72e432741
8 changed files with 24 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java vendored
View File

@ -69,7 +69,7 @@ public class UploadController implements UploadResource {
public FileUploadResult upload(@RequestPart(name = "file") MultipartFile file, public FileUploadResult upload(@RequestPart(name = "file") MultipartFile file,
@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(DOSSIER_ID) String dossierId,
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions, @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction) { @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) {
accessControlService.checkAccessPermissionsToDossier(dossierId); accessControlService.checkAccessPermissionsToDossier(dossierId);
if (file.getOriginalFilename() == null) { if (file.getOriginalFilename() == null) {
@ -81,7 +81,7 @@ public class UploadController implements UploadResource {
try { try {
switch (extension) { switch (extension) {
case "zip": case "zip":
return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticRedaction); return handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
case "csv": case "csv":
return uploadService.importCsv(dossierId, file.getBytes()); return uploadService.importCsv(dossierId, file.getBytes());
default: default:
@ -91,7 +91,7 @@ public class UploadController implements UploadResource {
if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) { if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
throw new NotAllowedException("Insufficient permissions"); throw new NotAllowedException("Insufficient permissions");
} }
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticRedaction); return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions, disableAutomaticAnalysis);
} }
} catch (IOException e) { } catch (IOException e) {
throw new BadRequestException(e.getMessage(), e); throw new BadRequestException(e.getMessage(), e);
@ -135,7 +135,7 @@ public class UploadController implements UploadResource {
} }
private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticRedaction) throws IOException { private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
File tempFile = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip"); File tempFile = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip");
try (var fileOutputStream = new FileOutputStream(tempFile)) { try (var fileOutputStream = new FileOutputStream(tempFile)) {
@ -145,7 +145,7 @@ public class UploadController implements UploadResource {
try { try {
checkForSymlinks(tempFile); checkForSymlinks(tempFile);
var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticRedaction); var zipData = unzip(tempFile, dossierId, keepManualRedactions, disableAutomaticAnalysis);
if (zipData.csvBytes != null) { if (zipData.csvBytes != null) {
try { try {
@ -188,7 +188,7 @@ public class UploadController implements UploadResource {
} }
private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticRedaction) throws IOException { private ZipData unzip(File tempFile, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException {
var zipData = new ZipData(); var zipData = new ZipData();
@ -199,7 +199,7 @@ public class UploadController implements UploadResource {
zipData.totalEntryArchive++; zipData.totalEntryArchive++;
if (!ze.isDirectory()) { if (!ze.isDirectory()) {
processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticRedaction); processFileZipEntry(ze, zipFile, dossierId, keepManualRedactions, zipData, disableAutomaticAnalysis);
} }
} }
} }
@ -207,7 +207,7 @@ public class UploadController implements UploadResource {
} }
private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticRedaction) throws IOException { private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData, boolean disableAutomaticAnalysis) throws IOException {
var extension = getExtension(ze.getName()); var extension = getExtension(ze.getName());
@ -242,7 +242,7 @@ public class UploadController implements UploadResource {
zipData.containedUnpermittedFiles = false; zipData.containedUnpermittedFiles = false;
try { try {
var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticRedaction); var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions, disableAutomaticAnalysis);
zipData.fileUploadResult.getFileIds().addAll(result.getFileIds()); zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
} catch (Exception e) { } catch (Exception e) {
log.debug("PDF File inside ZIP failed", e); log.debug("PDF File inside ZIP failed", e);

5
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/FileControllerV2.java vendored
View File

@ -38,7 +38,6 @@ import com.iqser.red.service.persistence.service.v2.api.external.model.FileDelet
import com.iqser.red.service.persistence.service.v2.api.external.model.FileStatusList; import com.iqser.red.service.persistence.service.v2.api.external.model.FileStatusList;
import com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource; import com.iqser.red.service.persistence.service.v2.api.external.resource.FileResource;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -62,11 +61,11 @@ public class FileControllerV2 implements FileResource {
@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestPart(name = FILE_PARAM) MultipartFile file, @RequestPart(name = FILE_PARAM) MultipartFile file,
@RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges, @RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges,
@RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction) { @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) {
dossierTemplateController.getDossierTemplate(dossierTemplateId); dossierTemplateController.getDossierTemplate(dossierTemplateId);
return uploadController.upload(file, dossierId, keepManualChanges, disableAutomaticRedaction); return uploadController.upload(file, dossierId, keepManualChanges, disableAutomaticAnalysis);
} }

4
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/UploadResource.java vendored
View File

@ -28,7 +28,7 @@ public interface UploadResource {
String IMPORT_REDACTIONS_PATH = ExternalApi.BASE_PATH + "/import-redactions"; String IMPORT_REDACTIONS_PATH = ExternalApi.BASE_PATH + "/import-redactions";
String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}"; String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}";
String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}"; String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}";
String DISABLE_AUTOMATIC_REDACTION_PARAM = "disableAutomaticRedaction"; String DISABLE_AUTOMATIC_ANALYSIS_PARAM = "disableAutomaticAnalysis";
@ResponseBody @ResponseBody
@ -40,7 +40,7 @@ public interface UploadResource {
FileUploadResult upload(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, FileUploadResult upload(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(DOSSIER_ID) String dossierId,
@RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions, @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction); @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic analysis for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis);
@ResponseBody @ResponseBody

4
persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/FileResource.java vendored
View File

@ -48,7 +48,7 @@ public interface FileResource {
String CREATE_DOWNLOAD_PATH = "/create-download"; String CREATE_DOWNLOAD_PATH = "/create-download";
String BULK_CREATE_DOWNLOAD_PATH = "/bulk/create-download"; String BULK_CREATE_DOWNLOAD_PATH = "/bulk/create-download";
String KEEP_MANUAL_CHANGES_PARAM = "keepManualChanges"; String KEEP_MANUAL_CHANGES_PARAM = "keepManualChanges";
String DISABLE_AUTOMATIC_REDACTION_PARAM = "disableAutomaticRedaction"; String DISABLE_AUTOMATIC_ANALYSIS_PARAM = "disableAutomaticAnalysis";
String DELETE_PERMANENTLY_PARAM = "deletePermanently"; String DELETE_PERMANENTLY_PARAM = "deletePermanently";
String FILE_PARAM = "file"; String FILE_PARAM = "file";
String FILE_ID_PARAM = "fileId"; String FILE_ID_PARAM = "fileId";
@ -64,7 +64,7 @@ public interface FileResource {
@Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier to where the file is to be uploaded.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId, @Parameter(name = DOSSIER_ID_PARAM, description = "The identifier of the dossier to where the file is to be uploaded.", required = true) @PathVariable(DOSSIER_ID_PARAM) String dossierId,
@Parameter(name = FILE_PARAM, description = "The file to be uploaded.", required = true) @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = FILE_PARAM) MultipartFile file, @Parameter(name = FILE_PARAM, description = "The file to be uploaded.", required = true) @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = FILE_PARAM) MultipartFile file,
@Parameter(name = KEEP_MANUAL_CHANGES_PARAM, description = "A Toggle to keep manual changes: Manual changes are manually added annotations or manipulations on existing ones. If set to `true` the system keeps the manual changes on re-uploading (overwriting) the file.") @RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges, @Parameter(name = KEEP_MANUAL_CHANGES_PARAM, description = "A Toggle to keep manual changes: Manual changes are manually added annotations or manipulations on existing ones. If set to `true` the system keeps the manual changes on re-uploading (overwriting) the file.") @RequestParam(value = KEEP_MANUAL_CHANGES_PARAM, required = false, defaultValue = "false") boolean keepManualChanges,
@Parameter(name = DISABLE_AUTOMATIC_REDACTION_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_REDACTION_PARAM, required = false, defaultValue = "false") boolean disableAutomaticRedaction); @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis);
@ResponseStatus(value = HttpStatus.OK) @ResponseStatus(value = HttpStatus.OK)

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java
View File

@ -42,7 +42,7 @@ public class FileService {
private final WebsocketService websocketService; private final WebsocketService websocketService;
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size, boolean disableAutomaticRedaction) { public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size, boolean disableAutomaticAnalysis) {
dossierPersistenceService.getAndValidateDossier(request.getDossierId()); dossierPersistenceService.getAndValidateDossier(request.getDossierId());
@ -65,7 +65,7 @@ public class FileService {
} else { } else {
// the file is new, should create a new status for it. // the file is new, should create a new status for it.
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename()); log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), size, disableAutomaticRedaction); fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), size, disableAutomaticAnalysis);
} }
return new JSONPrimitive<>(request.getFileId()); return new JSONPrimitive<>(request.getFileId());

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -287,7 +287,7 @@ public class FileStatusService {
.build(); .build();
if(fileEntity.isExcluded() && fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)){ if(fileEntity.isExcludedFromAutomaticAnalysis() && fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMPORTED_REDACTIONS)){
analyseRequest.setMessageType(MessageType.IMPORTED_REDACTIONS_ONLY); analyseRequest.setMessageType(MessageType.IMPORTED_REDACTIONS_ONLY);
} }
@ -618,9 +618,9 @@ public class FileStatusService {
@Transactional @Transactional
public void createStatus(String dossierId, String fileId, String uploader, String filename, long size, boolean disableAutomaticRedaction) { public void createStatus(String dossierId, String fileId, String uploader, String filename, long size, boolean disableAutomaticAnalysis) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size, disableAutomaticRedaction); fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size, disableAutomaticAnalysis);
websocketService.sendFileEvent(dossierId, fileId, FileEventType.CREATE); websocketService.sendFileEvent(dossierId, fileId, FileEventType.CREATE);
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
} }

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java
View File

@ -64,7 +64,7 @@ public class UploadService {
@PreAuthorize("hasAuthority('" + UPLOAD_FILE + "')") @PreAuthorize("hasAuthority('" + UPLOAD_FILE + "')")
public FileUploadResult processSingleFile(String dossierId, String fileName, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticRedaction) { public FileUploadResult processSingleFile(String dossierId, String fileName, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) {
dossierManagementService.getDossierById(dossierId, false, false); dossierManagementService.getDossierById(dossierId, false, false);
@ -76,7 +76,7 @@ public class UploadService {
try { try {
storageService.storeObject(TenantContext.getTenantId(), storageId, new ByteArrayInputStream(fileContent)); storageService.storeObject(TenantContext.getTenantId(), storageId, new ByteArrayInputStream(fileContent));
fileService.upload(new AddFileRequest(fileName, fileId, dossierId, KeycloakSecurity.getUserId()), keepManualRedactions, fileContent.length, disableAutomaticRedaction); fileService.upload(new AddFileRequest(fileName, fileId, dossierId, KeycloakSecurity.getUserId()), keepManualRedactions, fileContent.length, disableAutomaticAnalysis);
} catch (Exception e) { } catch (Exception e) {
storageService.deleteObject(TenantContext.getTenantId(), storageId); storageService.deleteObject(TenantContext.getTenantId(), storageId);

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -50,7 +50,7 @@ public class FileStatusPersistenceService {
private final EntityManager entityManager; private final EntityManager entityManager;
public FileEntity createStatus(String dossierId, String fileId, String filename, String uploader, long size, boolean disableAutomaticRedaction) { public FileEntity createStatus(String dossierId, String fileId, String filename, String uploader, long size, boolean disableAutomaticAnalysis) {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
FileEntity file = new FileEntity(); FileEntity file = new FileEntity();
@ -68,7 +68,7 @@ public class FileStatusPersistenceService {
file.setProcessingErrorCounter(0); file.setProcessingErrorCounter(0);
file.setFileSize(size); file.setFileSize(size);
file.setComponentMappingVersions(new ArrayList<>()); file.setComponentMappingVersions(new ArrayList<>());
file.setExcludedFromAutomaticAnalysis(disableAutomaticRedaction); file.setExcludedFromAutomaticAnalysis(disableAutomaticAnalysis);
return fileRepository.save(file); return fileRepository.save(file);
} }