Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #470: RED-4501: Sonar issues: persistence-service

Browse Source 
Merge in RED/persistence-service from RED-4501-ps1 to master

* commit '9f3f23ccc174b117119c529d54b29e5dbc1b4d84':
  RED-4501: Sonar issues: persistence-service
This commit is contained in:
Ali Oezyetimoglu 2022-07-04 14:09:37 +02:00 committed by Philipp Schramm
commit 126beb1815
3 changed files with 97 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java
View File

@ -115,7 +115,7 @@ public class JDBCWriteUtils {
}
private String toSnakeCase(String name) {
String ret = name.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2");
String ret = name.replaceAll("([A-Z]{1,100})([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2");
return ret.toLowerCase();
}

153
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -240,74 +240,6 @@ public class FileStatusService {
}
@Transactional
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
if(!fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)){
addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename());
return;
}
if (fileEntity.isExcluded()) {
log.debug("File {} is excluded", fileEntity.getId());
return;
}
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
boolean reanalyse = fileModel.isReanalysisRequired();
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId);
return;
}
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel);
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId);
return;
}
var analyseRequest = AnalyzeRequest.builder()
.messageType(messageType)
.dossierId(dossierId)
.analysisNumber(fileModel.getNumberOfAnalyses() + 1)
.sectionsToReanalyse(sectionsToReanalyse)
.fileId(fileId)
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed())
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build();
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
if (messageType.equals(MessageType.REANALYSE)) {
setStatusProcessing(fileId);
} else {
setStatusFullProcessing(fileId);
}
try {
if (priority) {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest));
} else {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest));
}
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
@Transactional
public void setStatusReprocess(String dossierId, String fileId, boolean priority) {
@ -491,7 +423,6 @@ public class FileStatusService {
}
@Transactional
public void deleteManualRedactions(String dossierId, String fileId) {
OffsetDateTime now = OffsetDateTime.now();
@ -553,6 +484,80 @@ public class FileStatusService {
}
public boolean hasChangesSince(String dossierId, OffsetDateTime since) {
return fileStatusPersistenceService.hasChangesSince(dossierId, since);
}
@Transactional
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
if(!fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)){
addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename());
return;
}
if (fileEntity.isExcluded()) {
log.debug("File {} is excluded", fileEntity.getId());
return;
}
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
boolean reanalyse = fileModel.isReanalysisRequired();
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId);
return;
}
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus(), fileModel);
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId);
return;
}
var analyseRequest = AnalyzeRequest.builder()
.messageType(messageType)
.dossierId(dossierId)
.analysisNumber(fileModel.getNumberOfAnalyses() + 1)
.sectionsToReanalyse(sectionsToReanalyse)
.fileId(fileId)
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed())
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build();
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
if (messageType.equals(MessageType.REANALYSE)) {
setStatusProcessing(fileId);
} else {
setStatusFullProcessing(fileId);
}
try {
if (priority) {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest));
} else {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest));
}
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
protected void addToNerQueue(String dossierId, String fileId) {
setStatusNerAnalyzing(fileId);
@ -588,9 +593,7 @@ public class FileStatusService {
} else if (manualRedactionProviderService.getManualRedactions(fileId)
.getResizeRedactions()
.stream()
.filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed()))
.findFirst()
.isPresent()) {
.anyMatch(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed()))) {
return MessageType.REANALYSE;
}
return MessageType.ANALYSE;
@ -635,10 +638,4 @@ public class FileStatusService {
return fileAttributeList;
}
public boolean hasChangesSince(String dossierId, OffsetDateTime since) {
return fileStatusPersistenceService.hasChangesSince(dossierId, since);
}
}

25
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackedArchiver.java
View File

@ -27,33 +27,44 @@ public class FileSystemBackedArchiver implements AutoCloseable {
private final File tempFile;
private final ZipOutputStream zipOutputStream;
@SneakyThrows
public FileSystemBackedArchiver() {
if(SystemUtils.IS_OS_UNIX) {
if (SystemUtils.IS_OS_UNIX) {
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
tempFile = Files.createTempFile("archive", ".zip", attr).toFile();
} else {
tempFile = Files.createTempFile("archive", ".zip").toFile();
tempFile.setReadable(true, true);
tempFile.setWritable(true, true);
tempFile.setExecutable(true, true);
boolean isReadable = tempFile.setReadable(true, true);
boolean isWritable = tempFile.setWritable(true, true);
boolean isExecutable = tempFile.setExecutable(true, true);
if (!isReadable || !isWritable || !isExecutable) {
log.warn("Could not archive temporary zip file.");
}
}
zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile));
}
public void addEntries(ArchiveModel... archives) {
addEntries(Arrays.asList(archives));
}
public void addEntries(List<ArchiveModel> archives) {
archives.forEach(this::addEntry);
}
@SneakyThrows
public InputStream toInputStream() {
try {
zipOutputStream.close();
} catch (IOException e) {
@ -62,6 +73,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
return new BufferedInputStream(new FileInputStream(tempFile));
}
@SneakyThrows
public void addEntry(ArchiveModel archiveModel) {
// begin writing a new ZIP entry, positions the stream to the start of the entry entity
@ -82,6 +94,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
@Override
public void close() {
try {
boolean res = tempFile.delete();
if (!res) {
@ -93,7 +106,9 @@ public class FileSystemBackedArchiver implements AutoCloseable {
}
}
public long getContentLength() {
try {
zipOutputStream.close();
} catch (IOException e) {
@ -102,6 +117,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
return tempFile.length();
}
@Data
@AllArgsConstructor
public static class ArchiveModel {
@ -109,6 +125,7 @@ public class FileSystemBackedArchiver implements AutoCloseable {
private String folderName;
private String name;
private byte[] bytes;
}
}