Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9147: Provide events via websocket

Browse Source
This commit is contained in:
Dominique Eifländer 2024-05-22 14:48:45 +02:00
parent edb333aa29
commit 7d875ee7eb
36 changed files with 447 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java vendored
View File

@ -18,6 +18,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEventType;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierCreatorService;
import org.apache.commons.lang3.StringUtils;
@ -41,6 +42,7 @@ import com.iqser.red.service.persistence.management.v1.processor.roles.Applicati
import com.iqser.red.service.persistence.management.v1.processor.service.AccessControlService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
@ -76,6 +78,7 @@ public class DossierController implements DossierResource {
private final AccessControlService accessControlService;
private final DossierACLService dossierACLService;
private final DossierCreatorService dossierCreatorService;
private final WebsocketService websocketService;
@Override
@ -217,6 +220,8 @@ public class DossierController implements DossierResource {
.target(Map.of("dossierId", dossierRequest.getDossierId()))
.build()));
websocketService.sendDossierEvent(dossierRequest.getDossierId(), DossierEventType.UPDATE);
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
return new ResponseEntity<>(updatedDossier, httpHeaders, HttpStatus.OK);
@ -231,6 +236,7 @@ public class DossierController implements DossierResource {
.message("Dossier has been created.")
.build());
websocketService.sendDossierEvent(created.getId(), DossierEventType.CREATE);
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
return new ResponseEntity<>(created, httpHeaders, HttpStatus.CREATED);
@ -387,6 +393,7 @@ public class DossierController implements DossierResource {
.target(Map.of("dossierId", dossierId, "dossierName", dossier.getDossierName()))
.build()));
websocketService.sendDossierEvent(dossierId, DossierEventType.SOFT_DELETE);
}
@ -476,6 +483,7 @@ public class DossierController implements DossierResource {
.category(AuditCategory.DOSSIER.name())
.message("Dossier archived.")
.build());
websocketService.sendDossierEvent(dossierId, DossierEventType.ARCHIVE);
}
}
@ -493,6 +501,7 @@ public class DossierController implements DossierResource {
.category(AuditCategory.DOSSIER.name())
.message("Dossier restored from archive.")
.build());
websocketService.sendDossierEvent(dossierId, DossierEventType.UNARCHIVE);
}
}
@ -513,7 +522,7 @@ public class DossierController implements DossierResource {
.category(AuditCategory.DOSSIER.name())
.message("Dossier permanently deleted.")
.build());
websocketService.sendDossierEvent(dossierId, DossierEventType.HARD_DELETE);
}
}
@ -533,6 +542,7 @@ public class DossierController implements DossierResource {
.category(AuditCategory.DOSSIER.name())
.message("Dossier restored from trash.")
.build());
websocketService.sendDossierEvent(dossierId, DossierEventType.UNDELETE);
}
}

5
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/FileAttributesController.java vendored
View File

@ -13,6 +13,9 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
@ -47,6 +50,7 @@ public class FileAttributesController implements FileAttributesResource {
private final AuditPersistenceService auditPersistenceService;
private final FileStatusService fileStatusService;
private final AccessControlService accessControlService;
private final WebsocketService websocketService;
@Override
@ -161,6 +165,7 @@ public class FileAttributesController implements FileAttributesResource {
}
accessControlService.verifyUserIsMemberOrApprover(dossierId);
fileAttributesManagementService.setFileAttributes(dossierId, fileId, fileAttributes.getAttributeIdToValue());
websocketService.sendFileEvent(dossierId, fileId, FileEventType.UPDATE);
auditPersistenceService.audit(AuditRequest.builder()
.userId(KeycloakSecurity.getUserId())
.objectId(fileId)

4
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -37,7 +37,7 @@ dependencies {
}
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.knecon.fforesight:database-tenant-commons:0.24.0")
api("com.knecon.fforesight:keycloak-commons:0.28.0")
api("com.knecon.fforesight:keycloak-commons:0.29.0")
api("com.knecon.fforesight:tracing-commons:0.5.0")
api("com.knecon.fforesight:swagger-commons:0.7.0")
api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0")
@ -47,6 +47,8 @@ dependencies {
api("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-websocket:${springBootStarterVersion}")
api("org.springframework.security:spring-security-messaging:6.1.3")
api("com.iqser.red.commons:spring-commons:2.1.0")
api("com.iqser.red.commons:jackson-commons:2.1.0")
api("com.iqser.red.commons:storage-commons:2.45.0")

95
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/WebSocketConfiguration.java Normal file
View File

@ -0,0 +1,95 @@
package com.iqser.red.service.persistence.management.v1.processor.configuration;
import java.util.Collections;
import java.util.Optional;
import org.apache.tomcat.websocket.server.WsSci;
import org.springframework.boot.web.embedded.tomcat.TomcatContextCustomizer;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.simp.config.ChannelRegistration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.messaging.simp.stomp.StompCommand;
import org.springframework.messaging.simp.stomp.StompHeaderAccessor;
import org.springframework.messaging.support.ChannelInterceptor;
import org.springframework.messaging.support.MessageHeaderAccessor;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.oauth2.server.resource.authentication.BearerTokenAuthenticationToken;
import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Configuration
@EnableWebSocketMessageBroker
@RequiredArgsConstructor
public class WebSocketConfiguration implements WebSocketMessageBrokerConfigurer {
private final TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@Override
public void configureMessageBroker(MessageBrokerRegistry config) {
config.enableSimpleBroker("/topic");
config.setApplicationDestinationPrefixes("/app");
}
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/redaction-gateway-v1/websocket").setAllowedOrigins("*");
}
@Override
public void configureClientInboundChannel(ChannelRegistration registration) {
// https://docs.spring.io/spring-framework/reference/web/websocket/stomp/authentication-token-based.html
registration.interceptors(new ChannelInterceptor() {
@Override
public Message<?> preSend(Message<?> message, MessageChannel channel) {
StompHeaderAccessor accessor = MessageHeaderAccessor.getAccessor(message, StompHeaderAccessor.class);
if (StompCommand.CONNECT.equals(accessor.getCommand())) {
Optional.ofNullable(accessor.getNativeHeader("Authorization"))
.ifPresent(ah -> {
String bearerToken = ah.get(0).replace("Bearer ", "");
log.debug("Received bearer token {}", bearerToken);
AuthenticationManager authenticationManager = tenantAuthenticationManagerResolver.resolve(bearerToken);
JwtAuthenticationToken token = (JwtAuthenticationToken) authenticationManager.authenticate(new BearerTokenAuthenticationToken(bearerToken));
accessor.setUser(token);
});
}
return message;
}
});
}
@Bean
public TomcatServletWebServerFactory tomcatContainerFactory() {
TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory();
factory.setTomcatContextCustomizers(Collections.singletonList(tomcatContextCustomizer()));
return factory;
}
@Bean
public TomcatContextCustomizer tomcatContextCustomizer() {
return context -> context.addServletContainerInitializer(new WsSci(), null);
}
}

23
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/WebSocketSecurityConfig.java Normal file
View File

@ -0,0 +1,23 @@
//package com.iqser.red.service.persistence.management.v1.processor.configuration;
//
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.messaging.Message;
//import org.springframework.security.authorization.AuthorizationManager;
//import org.springframework.security.config.annotation.web.socket.EnableWebSocketSecurity;
//import org.springframework.security.messaging.access.intercept.MessageMatcherDelegatingAuthorizationManager;
//
//@Configuration
//@EnableWebSocketSecurity
//public class WebSocketSecurityConfig {
//
// @Bean
// AuthorizationManager<Message<?>> messageAuthorizationManager(MessageMatcherDelegatingAuthorizationManager.Builder messages) {
//
// messages
// .simpDestMatchers("/redaction-gateway-v1/websocket/**", "/redaction-gateway-v1/websocket/**/**").authenticated()
// .anyMessage().authenticated();
//
// return messages.build();
// }
//}

32
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/AnalyseEvent.java Normal file
View File

@ -0,0 +1,32 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
import java.time.OffsetDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class AnalyseEvent {
private String dossierId;
private String fileId;
private AnalyseStatus analyseStatus;
private Integer analysisNumber;
private OffsetDateTime timestamp;
private int numberOfPagesToOCR;
private int numberOfOCRedPages;
public AnalyseEvent(String dossierId, String fileId, AnalyseStatus analyseStatus, Integer analysisNumber, OffsetDateTime timestamp) {
this.dossierId = dossierId;
this.fileId = fileId;
this.analyseStatus = analyseStatus;
this.analysisNumber = analysisNumber;
this.timestamp = timestamp;
}
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/AnalyseStatus.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
public enum AnalyseStatus {
READ_ONLY_PROCESSING,
PROCESSING,
OCR_PROCESSING,
LAYOUT_UPDATE,
FINISHED,
ERROR
}

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DossierEvent.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DossierEvent {
private String dossierId;
private DossierEventType dossierEventType;
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DossierEventType.java Normal file
View File

@ -0,0 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
public enum DossierEventType {
CREATE, UPDATE, SOFT_DELETE, HARD_DELETE, UNDELETE, ARCHIVE, UNARCHIVE
}

19
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/DownloadEvent.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DownloadEvent {
private String downloadId;
private String userId;
private DownloadStatusValue status;
}

19
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/FileEvent.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileEvent {
private String dossierId;
private String fileId;
private FileEventType fileEventType;
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/FileEventType.java Normal file
View File

@ -0,0 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
public enum FileEventType {
CREATE, UPDATE, SOFT_DELETE, HARD_DELETE, UNDELETE
}

15
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/websocket/NotificationEvent.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.model.websocket;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class NotificationEvent {
private String userId;
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java
View File

@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@ -35,6 +36,7 @@ public class AnalysisFlagsCalculationService {
EntityLogService entityLogService;
ViewedPagesPersistenceService viewedPagesPersistenceService;
ObservationRegistry observationRegistry;
WebsocketService websocketService;
@Timed("redactmanager_calculateFlags")
@ -137,6 +139,7 @@ public class AnalysisFlagsCalculationService {
}
fileStatusPersistenceService.setLastFlagCalculation(fileId, OffsetDateTime.now());
websocketService.sendFileEvent(dossierId, fileId, FileEventType.UPDATE);
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DownloadService.java
View File

@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@ -40,7 +41,7 @@ public class DownloadService {
private final FileStatusPersistenceService fileStatusPersistenceService;
private final DossierPersistenceService dossierPersistenceService;
private final ReportTemplateRepository reportTemplateRepository;
private final ObjectMapper objectMapper;
private final WebsocketService websocketService;
private final RabbitTemplate rabbitTemplate;
@ -87,6 +88,7 @@ public class DownloadService {
request.getDownloadFileTypes(),
request.getReportTemplateIds(),
request.getRedactionPreviewColor());
websocketService.sendDownloadEvent(storageId, request.getUserId(), DownloadStatusValue.QUEUED);
addToDownloadQueue(DownloadJob.builder().storageId(storageId).userId(request.getUserId()).includeUnprocessed(request.getIncludeUnprocessed()).build(), 1);
return new JSONPrimitive<>(storageId);

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java
View File

@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Confl
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
@ -48,6 +49,7 @@ public class FileService {
private final DossierPersistenceService dossierPersistenceService;
private final DossierService dossierService;
private final FileDeletionService fileDeletionService;
private final WebsocketService websocketService;
public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions, long size) {
@ -102,6 +104,7 @@ public class FileService {
OffsetDateTime softDeleteTime = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileDeletionService.softDeleteFiles(List.of(fileId), softDeleteTime);
fileDeletionService.reindexDeletedFiles(dossierId, List.of(fileId));
websocketService.sendFileEvent(dossierId, fileId, FileEventType.SOFT_DELETE);
}
@ -115,6 +118,9 @@ public class FileService {
fileDeletionService.hardDeleteFiles(fileIds);
fileDeletionService.hardDeleteFileDataAndIndexUpdates(dossierId, fileIds);
fileIds.forEach(fileId -> {
websocketService.sendFileEvent(dossierId, fileId, FileEventType.HARD_DELETE);
});
}
@ -131,6 +137,9 @@ public class FileService {
fileDeletionService.undeleteFile(fileId, softDeletedTime);
}
fileDeletionService.reindexUndeletedFiles(dossier.getDossierTemplateId(), dossierId, fileIds);
fileIds.forEach(fileId -> {
websocketService.sendFileEvent(dossierId, fileId, FileEventType.UNDELETE);
});
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java
View File

@ -47,7 +47,7 @@ public class FileStatusProcessingUpdateService {
default:
retryTemplate.execute(retryContext -> {
log.info("Analysis Successful for dossier {} and file {}, Attempt to update status: {}", dossierId, fileId, retryContext.getRetryCount());
fileStatusService.setStatusSuccessful(fileId, analyzeResult);
fileStatusService.setStatusSuccessful(dossierId, fileId, analyzeResult);
return null;
});
@ -110,7 +110,7 @@ public class FileStatusProcessingUpdateService {
fileId,
dossierId,
fileErrorInfo != null ? fileErrorInfo.getCause() : null);
fileStatusService.setStatusError(fileId, fileErrorInfo);
fileStatusService.setStatusError(dossierId, fileId, fileErrorInfo);
return null;
});
}

52
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -11,6 +11,8 @@ import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
@ -88,7 +90,7 @@ public class FileStatusService {
ViewedPagesPersistenceService viewedPagesPersistenceService;
FileManagementServiceSettings fileManagementServiceSettings;
LayoutParsingRequestFactory layoutParsingRequestFactory;
WebsocketService websocketService;
@Transactional
public List<FileModel> getAllRelevantStatusesForReanalysisScheduler() {
@ -173,8 +175,10 @@ public class FileStatusService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)) {
addToPreprocessingQueue(dossierId, fileId, fileEntity.getFilename());
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
@ -187,24 +191,28 @@ public class FileStatusService {
if (settings.isVisualLayoutParsingEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.VISUAL_LAYOUT)) {
log.info("Add file: {} from dossier {} to Visual Layout Parsing queue", fileId, dossierId);
addToVisualLayoutParsingQueue(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
if (settings.isFigureDetectionEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.FIGURE)) {
log.debug("Add file: {} from dossier {} to Figure Detection queue", fileId, dossierId);
addToFigureDetectionRequestQueue(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
if (settings.isCvTableParsingEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.TABLES)) {
log.debug("Add file: {} from dossier {} to Cv Service queue", fileId, dossierId);
addToTableParsingRequestQueue(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
if (settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.info("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
@ -215,6 +223,7 @@ public class FileStatusService {
if (dossierTemplate.isOcrByDefault() && fileModel.getOcrEndTime() == null && !fileModel.isSoftOrHardDeleted()) {
log.debug("Add file: {} from dossier {} to OCR queue", fileId, dossierId);
setStatusOcrQueued(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
@ -222,12 +231,14 @@ public class FileStatusService {
var layoutParsingRequest = layoutParsingRequestFactory.build(dossierId, fileId, priority);
setStatusFullProcessing(fileId);
rabbitTemplate.convertAndSend(LAYOUT_PARSING_REQUEST_QUEUE, layoutParsingRequest);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
}
@ -259,6 +270,20 @@ public class FileStatusService {
} else {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, analyseRequest);
}
sendAnalysisEvent(dossierId, fileId, fileEntity);
}
private void sendAnalysisEvent(String dossierId, String fileId, FileEntity fileEntity){
if(fileEntity.getProcessingStatus().equals(ProcessingStatus.UNPROCESSED) || fileEntity.getProcessingStatus().equals(ProcessingStatus.ERROR) || fileEntity.getProcessingStatus().equals(ProcessingStatus.REPROCESS) || fileEntity.getProcessingStatus().equals(ProcessingStatus.PROCESSED)){
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.PROCESSING, fileEntity.getNumberOfAnalyses() + 1);
}
}
private void sendReadOnlyAnalysisEvent(String dossierId, String fileId, FileEntity fileEntity){
if(fileEntity.getProcessingStatus().equals(ProcessingStatus.UNPROCESSED) || fileEntity.getProcessingStatus().equals(ProcessingStatus.ERROR) || fileEntity.getProcessingStatus().equals(ProcessingStatus.REPROCESS) || fileEntity.getProcessingStatus().equals(ProcessingStatus.PROCESSED) || fileEntity.getProcessingStatus().equals(ProcessingStatus.OCR_PROCESSING)){
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.READ_ONLY_PROCESSING, fileEntity.getNumberOfAnalyses() + 1);
}
}
@ -363,6 +388,7 @@ public class FileStatusService {
updateOCRStartTime(fileId);
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING_QUEUED);
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.OCR_PROCESSING, fileStatus.getNumberOfAnalyses() + 1);
addToOcrQueue(dossierId, fileId, 2);
}
@ -480,9 +506,9 @@ public class FileStatusService {
}
public void setStatusSuccessful(String fileId, AnalyzeResult analyzeResult) {
public void setStatusSuccessful(String dossierId, String fileId, AnalyzeResult analyzeResult) {
fileStatusPersistenceService.updateProcessingStatus(fileId,
fileStatusPersistenceService.updateProcessingStatus(dossierId, fileId,
analyzeResult.getNumberOfPages(),
analyzeResult.getDictionaryVersion(),
analyzeResult.getRulesVersion(),
@ -504,6 +530,7 @@ public class FileStatusService {
public void setApprovalStatusSuccessful(String dossierId, String fileId, WorkflowStatus workflowStatus) {
fileStatusPersistenceService.updateWorkflowStatus(fileId, workflowStatus, true);
websocketService.sendFileEvent(dossierId, fileId, FileEventType.UPDATE);
}
@ -559,13 +586,15 @@ public class FileStatusService {
public void createStatus(String dossierId, String fileId, String uploader, String filename, long size) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, size);
websocketService.sendFileEvent(dossierId, fileId, FileEventType.CREATE);
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
}
public void setStatusIndexingSuccessful(String fileId) {
fileStatusPersistenceService.setUpdateStatusIndexingSuccessful(fileId);
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
}
@ -575,22 +604,26 @@ public class FileStatusService {
}
public void setStatusError(String fileId, FileErrorInfo fileErrorInfo) {
public void setStatusError(String dossierId, String fileId, FileErrorInfo fileErrorInfo) {
fileStatusPersistenceService.updateStatusErrorInfo(fileId, fileErrorInfo);
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ERROR);
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.ERROR, 0);
}
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate) {
fileStatusPersistenceService.updateFileModificationDate(fileId, fileManipulationDate);
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
}
public void updateHasHighlights(String fileId, boolean hasHighlights) {
fileStatusPersistenceService.updateHasHighlights(fileId, hasHighlights);
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
}
@ -607,6 +640,7 @@ public class FileStatusService {
}
fileStatusPersistenceService.setAssignee(fileId, assignee, lastReviewer, lastApprover);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
}
@ -614,6 +648,8 @@ public class FileStatusService {
public void toggleExclusion(String dossierId, String fileId, boolean excluded) {
fileStatusPersistenceService.toggleExclusion(fileId, excluded);
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
if (!excluded) {
setStatusFullReprocess(dossierId, fileId, false, true);
@ -648,6 +684,8 @@ public class FileStatusService {
public void toggleAutomaticAnalysis(String dossierId, String fileId, boolean excludedFromAutomaticAnalysis) {
fileStatusPersistenceService.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis);
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
if (!excludedFromAutomaticAnalysis) {
setStatusFullReprocess(dossierId, fileId, false, true);
@ -691,6 +729,8 @@ public class FileStatusService {
viewedPagesPersistenceService.deleteForFile(fileId);
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
websocketService.sendFileEvent(fileStatus.getDossierId(), fileId, FileEventType.UPDATE);
addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
}

63
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/WebsocketService.java Normal file
View File

@ -0,0 +1,63 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseEvent;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEvent;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DossierEventType;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.DownloadEvent;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEvent;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.NotificationEvent;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class WebsocketService {
private final SimpMessagingTemplate template;
public void sendAnalysisEvent(String dossierId, String fileId, AnalyseStatus analyseStatus, int analysisNumber) {
template.convertAndSend("/topic/" + TenantContext.getTenantId() + "/analysis-events", new AnalyseEvent(dossierId, fileId, analyseStatus, analysisNumber, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
}
public void sendOcrUpdateAnalysisEvent(String dossierId, String fileId, AnalyseStatus analyseStatus, int analysisNumber, int numberOfPagesToOCR, int numberOfOCRedPages) {
template.convertAndSend("/topic/" + TenantContext.getTenantId() + "/analysis-events", new AnalyseEvent(dossierId, fileId, analyseStatus, analysisNumber, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), numberOfPagesToOCR, numberOfOCRedPages));
}
public void sendDownloadEvent(String downloadId, String userId, DownloadStatusValue downloadStatusValue){
template.convertAndSend("/topic/" + TenantContext.getTenantId() + "/download-events", new DownloadEvent(downloadId, userId, downloadStatusValue));
}
public void sendFileEvent(String dossierId, String fileId, FileEventType fileEventType){
template.convertAndSend("/topic/" + TenantContext.getTenantId() + "/file-events", new FileEvent(dossierId, fileId, fileEventType));
}
public void sendDossierEvent(String dossierId, DossierEventType dossierEventType){
template.convertAndSend("/topic/" + TenantContext.getTenantId() + "/dossier-events", new DossierEvent(dossierId, dossierEventType));
}
public void sendNotificationEvent(String userId){
template.convertAndSend("/topic/" + TenantContext.getTenantId() + "/notification-events", new NotificationEvent(userId));
}
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java
View File

@ -5,6 +5,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
@ -19,6 +20,7 @@ public class DownloadCompressingService {
private final DownloadPreparationService downloadPreparationService;
private final DownloadStatusRepository downloadStatusRepository;
private final RabbitTemplate rabbitTemplate;
private final WebsocketService websocketService;
public void markDownloadForCompression(String downloadId, String userId) {
@ -26,6 +28,7 @@ public class DownloadCompressingService {
var updated = downloadStatusRepository.updateStatusOnlyIfNotAlreadySet(downloadId, DownloadStatusValue.COMPRESSING);
if (updated == 1) {
websocketService.sendDownloadEvent(downloadId, userId, DownloadStatusValue.COMPRESSING);
rabbitTemplate.convertAndSend(MessagingConfiguration.DOWNLOAD_COMPRESSION_QUEUE, DownloadJob.builder().storageId(downloadId).userId(userId).build());
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadDLQMessageReceiver.java
View File

@ -10,6 +10,7 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
@ -28,6 +29,7 @@ public class DownloadDLQMessageReceiver {
private final RetryTemplate retryTemplate;
private final ObjectMapper objectMapper;
private final static int MAX_ERROR_CAUSE_LENGTH = 1024;
private final WebsocketService websocketService;
@RabbitListener(queues = MessagingConfiguration.DOWNLOAD_COMPRESSION_DLQ)
@ -85,6 +87,7 @@ public class DownloadDLQMessageReceiver {
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED, "error occured in unknown queue");
return null;
});
websocketService.sendDownloadEvent(downloadId, userId, DownloadStatusValue.FAILED);
}
@ -95,6 +98,7 @@ public class DownloadDLQMessageReceiver {
downloadStatusPersistenceService.updateStatus(downloadId, DownloadStatusValue.FAILED, errorCause);
return null;
});
websocketService.sendDownloadEvent(downloadId, userId, DownloadStatusValue.FAILED);
}
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.download
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
@ -63,6 +64,7 @@ public class DownloadPreparationService {
FileManagementServiceSettings settings;
DossierTemplatePersistenceService dossierTemplatePersistenceService;
DownloadRedactionFileStatusRepository downloadRedactionFileStatusRepository;
WebsocketService websocketService;
@Transactional
@ -220,6 +222,7 @@ public class DownloadPreparationService {
private void updateStatusToReady(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
downloadStatusPersistenceService.updateStatus(downloadStatus, DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
websocketService.sendDownloadEvent(downloadStatus.getStorageId(), downloadStatus.getUserId(), DownloadStatusValue.READY);
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadProcessorService.java
View File

@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.iqser.red.service.redaction.report.v1.api.model.ReportRequestMessage;
@ -28,7 +29,7 @@ public class DownloadProcessorService {
private final DownloadStatusPersistenceService downloadStatusPersistenceService;
private final DossierService dossierService;
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final WebsocketService websocketService;
@Transactional
@ -36,6 +37,8 @@ public class DownloadProcessorService {
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId());
downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING);
websocketService.sendDownloadEvent(downloadJob.getStorageId(), downloadJob.getUserId(), DownloadStatusValue.GENERATING);
var dossier = dossierService.getDossierById(downloadStatus.getFiles()
.iterator().next().getDossierId());

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -58,6 +58,7 @@ public class DownloadStatusPersistenceService {
downloadStatus.setDownloadFileTypes(downloadFileTypes != null ? new HashSet<>(downloadFileTypes) : new HashSet<>());
downloadStatus.setReports(reportTemplateIds != null ? reportTemplateRepository.findAllById(reportTemplateIds) : new ArrayList<>());
downloadStatus.setRedactionPreviewColor(redactionPreviewColor);
downloadStatus.setStatus(DownloadStatusValue.QUEUED);
downloadStatusRepository.save(downloadStatus);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -18,6 +18,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier;
import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@ -39,6 +41,7 @@ public class FileStatusPersistenceService {
private final FileAttributesRepository fileAttributesRepository;
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
private final DossierPersistenceService dossierService;
private final WebsocketService websocketService;
public void createStatus(String dossierId, String fileId, String filename, String uploader, long size) {
@ -108,7 +111,8 @@ public class FileStatusPersistenceService {
@Transactional
public void updateProcessingStatus(String fileId,
public void updateProcessingStatus(String dossierId,
String fileId,
int numberOfPages,
long dictionaryVersion,
long rulesVersion,
@ -136,6 +140,8 @@ public class FileStatusPersistenceService {
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
analysisNumber,
calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber);
}
@ -420,12 +426,13 @@ public class FileStatusPersistenceService {
}
}
public void softDeleteFiles(List<String> fileIds, OffsetDateTime softDeletedTime) {
fileRepository.softDeleteFiles(fileIds, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime);
}
public void hardDeleteFiles(List<String> fileIds) {
var now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/NotificationPersistenceService.java
View File

@ -10,6 +10,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.NotificationPreferencesService;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.EmailNotificationType;
@ -28,6 +29,8 @@ public class NotificationPersistenceService {
private final NotificationEmailService notificationEmailService;
private final WebsocketService websocketService;
public boolean hasNewNotificationsSince(String userId, OffsetDateTime since) {
@ -56,6 +59,7 @@ public class NotificationPersistenceService {
}
notificationRepository.save(notification);
websocketService.sendNotificationEvent(notification.getUserId());
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java
View File

@ -4,8 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
@ -35,6 +37,7 @@ public class LayoutParsingFinishedMessageReceiver {
private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService;
private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
private final SaasMigrationService saasMigrationService;
private final WebsocketService websocketService;
@SneakyThrows
@ -47,11 +50,15 @@ public class LayoutParsingFinishedMessageReceiver {
return;
}
var dossierId = layoutParsingRequestIdentifierService.parseDossierId(response.identifier());
var fileId = layoutParsingRequestIdentifierService.parseFileId(response.identifier());
fileStatusService.setStatusAnalyse(layoutParsingRequestIdentifierService.parseDossierId(response.identifier()),
layoutParsingRequestIdentifierService.parseFileId(response.identifier()),
layoutParsingRequestIdentifierService.parsePriority(response.identifier()));
fileStatusService.updateLayoutProcessedTime(layoutParsingRequestIdentifierService.parseFileId(response.identifier()));
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.LAYOUT_UPDATE, fileStatusService.getStatus(fileId).getNumberOfAnalyses() + 1);
log.info("Received message {} in {}", response, LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE);
}

16
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/OCRProcessingMessageReceiver.java
View File

@ -12,11 +12,14 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.model.OCRStatusUpdateResponse;
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -29,16 +32,25 @@ public class OCRProcessingMessageReceiver {
private final ObjectMapper objectMapper;
private final FileStatusService fileStatusService;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final WebsocketService websocketService;
@SneakyThrows
@RabbitListener(queues = MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE)
public void handleOCRStatusUpdateMessage(OCRStatusUpdateResponse response) {
var fileModel = fileStatusService.getStatus(response.getFileId());
if (response.isOcrStarted()) {
fileStatusProcessingUpdateService.ocrProcessingUpdateOnly(response.getFileId());
} else {
fileStatusService.updateOCRStatus(response);
websocketService.sendOcrUpdateAnalysisEvent(fileModel.getDossierId(),
fileModel.getId(),
AnalyseStatus.OCR_PROCESSING,
fileModel.getNumberOfAnalyses() + 1,
response.getNumberOfPagesToOCR(),
response.getNumberOfOCRedPages());
}
log.info("Received message {} in {}", response, MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE);
@ -77,8 +89,8 @@ public class OCRProcessingMessageReceiver {
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateService.requeueOCROrMarkFailed(ocrRequestMessage.getDossierId(),
ocrRequestMessage.getFileId(),
new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
ocrRequestMessage.getFileId(),
new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service", timestamp));
}
}

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -1,8 +1,11 @@
package com.iqser.red.service.peristence.v1.server;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import org.apache.catalina.Context;
import org.apache.tomcat.websocket.server.WsSci;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
@ -18,6 +21,8 @@ import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfigurati
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.boot.web.embedded.tomcat.TomcatContextCustomizer;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;

2
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml
View File

@ -135,7 +135,7 @@ bucket4j:
fforesight:
keycloak:
ignored-endpoints: [ '/redaction-gateway-v1','/actuator/health/**', '/redaction-gateway-v1/async/download/with-ott/**',
ignored-endpoints: [ '/redaction-gateway-v1', '/actuator/health/**',"/redaction-gateway-v1/websocket","/redaction-gateway-v1/websocket/**", '/redaction-gateway-v1/async/download/with-ott/**',
'/internal-api/**', '/redaction-gateway-v1/docs/swagger-ui',
'/redaction-gateway-v1/docs/**','/redaction-gateway-v1/docs',
'/api', '/api/','/api/docs/**','/api/docs','/api/docs/swagger-ui' ]

4
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -103,7 +103,7 @@ public class FileTesterAndProvider {
}
public void markFileAsProcessed(String fileId) {
public void markFileAsProcessed(String dossierId, String fileId) {
AnalyzeResult result = new AnalyzeResult();
result.setFileId(fileId);
@ -115,7 +115,7 @@ public class FileTesterAndProvider {
result.setDossierDictionaryVersion(1);
result.setAnalysisNumber(1);
result.setAnalysisVersion(1);
fileStatusService.setStatusSuccessful(fileId, result);
fileStatusService.setStatusSuccessful(dossierId, fileId, result);
fileStatusService.setStatusProcessed(fileId);
}

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java
View File

@ -298,7 +298,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k);
if (k == 1) {
fileStatusPersistenceService.updateProcessingStatus(fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1);
fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1);
reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k))));
}
if (k == 2) {

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
View File

@ -253,7 +253,7 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
public void forwardFileToApprovedState() {
fileTesterAndProvider.markFileAsProcessed(getFileId());
fileTesterAndProvider.markFileAsProcessed(getDossierId(), getFileId());
fileClient.setStatusApproved(getDossierId(), getFileId());

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadTest.java
View File

@ -81,7 +81,7 @@ public class DownloadTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var file2 = fileTesterAndProvider.testAndProvideFile(dossier, "file2");
fileTesterAndProvider.markFileAsProcessed(file2.getFileId());
fileTesterAndProvider.markFileAsProcessed(dossier.getId(), file2.getFileId());
fileClient.setStatusApproved(dossier.getId(), file2.getId());
var file22 = fileClient.getFileStatus(dossier.getId(), file2.getId());

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -291,7 +291,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
assertThat(loadedFile.getLastApprover()).isEqualTo(userId);
fileTesterAndProvider.markFileAsProcessed(file.getFileId());
fileTesterAndProvider.markFileAsProcessed(dossier.getId(), file.getFileId());
fileClient.setStatusApproved(dossier.getId(), file.getId());
loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED);

3
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -110,6 +110,7 @@ import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
import com.knecon.fforesight.mongo.database.commons.liquibase.MongoTenantCreatedEvent;
import com.knecon.fforesight.mongo.database.commons.liquibase.MongoTenantCreatedListener;
import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService;
@ -251,6 +252,8 @@ public abstract class AbstractPersistenceServerServiceTest {
protected CommentRepository commentRepository;
@Autowired
protected ResizeRedactionRepository resizeRedactionRepository;
@MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
private static String[] getAllRoles() {