diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java index 2b7d18a7d..8fc9549a8 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java @@ -115,7 +115,7 @@ public class JDBCWriteUtils { } private String toSnakeCase(String name) { - String ret = name.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2"); + String ret = name.replaceAll("([A-Z]{1,100})([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2"); return ret.toLowerCase(); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 43e868e20..55e0dd174 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -240,74 +240,6 @@ public class FileStatusService { } - @Transactional - protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse) { - - var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - var fileEntity = fileStatusPersistenceService.getStatus(fileId); - - if(!fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)){ - addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename()); - return; - } - - if (fileEntity.isExcluded()) { - log.debug("File {} is excluded", fileEntity.getId()); - return; - } - - var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); - reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); - - boolean reanalyse = fileModel.isReanalysisRequired(); - - if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { - log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); - addToImageQueue(dossierId, fileId); - return; - } - - MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel); - - if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { - log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); - addToNerQueue(dossierId, fileId); - return; - } - - var analyseRequest = AnalyzeRequest.builder() - .messageType(messageType) - .dossierId(dossierId) - .analysisNumber(fileModel.getNumberOfAnalyses() + 1) - .sectionsToReanalyse(sectionsToReanalyse) - .fileId(fileId) - .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) - .dossierTemplateId(dossier.getDossierTemplateId()) - .lastProcessed(fileModel.getLastProcessed()) - .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) - .excludedPages(fileModel.getExcludedPages()) - .build(); - - log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); - - if (messageType.equals(MessageType.REANALYSE)) { - setStatusProcessing(fileId); - } else { - setStatusFullProcessing(fileId); - } - - try { - if (priority) { - rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest)); - } else { - rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest)); - } - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - @Transactional public void setStatusReprocess(String dossierId, String fileId, boolean priority) { @@ -491,7 +423,6 @@ public class FileStatusService { } - @Transactional public void deleteManualRedactions(String dossierId, String fileId) { OffsetDateTime now = OffsetDateTime.now(); @@ -553,6 +484,80 @@ public class FileStatusService { } + public boolean hasChangesSince(String dossierId, OffsetDateTime since) { + + return fileStatusPersistenceService.hasChangesSince(dossierId, since); + } + + + @Transactional + protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse) { + + var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + + if(!fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)){ + addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename()); + return; + } + + if (fileEntity.isExcluded()) { + log.debug("File {} is excluded", fileEntity.getId()); + return; + } + + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); + reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); + + boolean reanalyse = fileModel.isReanalysisRequired(); + + if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) { + log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); + addToImageQueue(dossierId, fileId); + return; + } + + MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel); + + if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) { + log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); + addToNerQueue(dossierId, fileId); + return; + } + + var analyseRequest = AnalyzeRequest.builder() + .messageType(messageType) + .dossierId(dossierId) + .analysisNumber(fileModel.getNumberOfAnalyses() + 1) + .sectionsToReanalyse(sectionsToReanalyse) + .fileId(fileId) + .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) + .dossierTemplateId(dossier.getDossierTemplateId()) + .lastProcessed(fileModel.getLastProcessed()) + .fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId())) + .excludedPages(fileModel.getExcludedPages()) + .build(); + + log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType); + + if (messageType.equals(MessageType.REANALYSE)) { + setStatusProcessing(fileId); + } else { + setStatusFullProcessing(fileId); + } + + try { + if (priority) { + rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest)); + } else { + rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest)); + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + protected void addToNerQueue(String dossierId, String fileId) { setStatusNerAnalyzing(fileId); @@ -588,9 +593,7 @@ public class FileStatusService { } else if (manualRedactionProviderService.getManualRedactions(fileId) .getResizeRedactions() .stream() - .filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed())) - .findFirst() - .isPresent()) { + .anyMatch(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed()))) { return MessageType.REANALYSE; } return MessageType.ANALYSE; @@ -635,10 +638,4 @@ public class FileStatusService { return fileAttributeList; } - - public boolean hasChangesSince(String dossierId, OffsetDateTime since) { - - return fileStatusPersistenceService.hasChangesSince(dossierId, since); - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackedArchiver.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackedArchiver.java index 63ececd41..1c0e408a6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackedArchiver.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackedArchiver.java @@ -27,33 +27,44 @@ public class FileSystemBackedArchiver implements AutoCloseable { private final File tempFile; private final ZipOutputStream zipOutputStream; + @SneakyThrows public FileSystemBackedArchiver() { - if(SystemUtils.IS_OS_UNIX) { + + if (SystemUtils.IS_OS_UNIX) { FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); tempFile = Files.createTempFile("archive", ".zip", attr).toFile(); } else { tempFile = Files.createTempFile("archive", ".zip").toFile(); - tempFile.setReadable(true, true); - tempFile.setWritable(true, true); - tempFile.setExecutable(true, true); + boolean isReadable = tempFile.setReadable(true, true); + boolean isWritable = tempFile.setWritable(true, true); + boolean isExecutable = tempFile.setExecutable(true, true); + + if (!isReadable || !isWritable || !isExecutable) { + log.warn("Could not archive temporary zip file."); + } } zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile)); } + public void addEntries(ArchiveModel... archives) { + addEntries(Arrays.asList(archives)); } + public void addEntries(List archives) { archives.forEach(this::addEntry); } + @SneakyThrows public InputStream toInputStream() { + try { zipOutputStream.close(); } catch (IOException e) { @@ -62,6 +73,7 @@ public class FileSystemBackedArchiver implements AutoCloseable { return new BufferedInputStream(new FileInputStream(tempFile)); } + @SneakyThrows public void addEntry(ArchiveModel archiveModel) { // begin writing a new ZIP entry, positions the stream to the start of the entry entity @@ -82,6 +94,7 @@ public class FileSystemBackedArchiver implements AutoCloseable { @Override public void close() { + try { boolean res = tempFile.delete(); if (!res) { @@ -93,7 +106,9 @@ public class FileSystemBackedArchiver implements AutoCloseable { } } + public long getContentLength() { + try { zipOutputStream.close(); } catch (IOException e) { @@ -102,6 +117,7 @@ public class FileSystemBackedArchiver implements AutoCloseable { return tempFile.length(); } + @Data @AllArgsConstructor public static class ArchiveModel { @@ -109,6 +125,7 @@ public class FileSystemBackedArchiver implements AutoCloseable { private String folderName; private String name; private byte[] bytes; + } }