From 0307da9093940d34b45e37cc700b2c730252ed84 Mon Sep 17 00:00:00 2001 From: devplant Date: Thu, 25 May 2023 14:20:14 +0300 Subject: [PATCH 1/2] RED-6072 - As Operation I want to see why files are in an ERROR state update from iqser to knecon --- bamboo-specs/src/main/java/buildjob/PlanSpec.java | 4 ++-- bamboo-specs/src/main/resources/scripts/build-java.sh | 2 +- bamboo-specs/src/main/resources/scripts/sonar-java.sh | 4 ++-- ocr-service-image-v1/pom.xml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java index ec40b3a..b401360 100644 --- a/bamboo-specs/src/main/java/buildjob/PlanSpec.java +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -95,7 +95,7 @@ public class PlanSpec { .enabled(true), new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository()) - .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .dockerConfiguration(new DockerConfiguration().image("nexus.knecon.com:5001/infra/maven:3.8.4-openjdk-17-slim") .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") .volume("/var/run/docker.sock", "/var/run/docker.sock")))) .linkedRepositories("RED / " + SERVICE_NAME) @@ -113,7 +113,7 @@ public class PlanSpec { .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository").checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME)) - .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0") + .dockerConfiguration(new DockerConfiguration().image("nexus.knecon.com:5001/infra/maven:3.6.2-jdk-13-3.0.0") .dockerRunArguments("--net=host") .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") .volume("/var/run/docker.sock", "/var/run/docker.sock")))) diff --git a/bamboo-specs/src/main/resources/scripts/build-java.sh b/bamboo-specs/src/main/resources/scripts/build-java.sh index 4f8870c..29f86b1 100755 --- a/bamboo-specs/src/main/resources/scripts/build-java.sh +++ b/bamboo-specs/src/main/resources/scripts/build-java.sh @@ -51,7 +51,7 @@ mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \ -Dmaven.wagon.http.ssl.insecure=true \ -Dmaven.wagon.http.ssl.allowall=true \ -Dmaven.wagon.http.ssl.ignore.validity.dates=true \ - -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases + -DaltDeploymentRepository=iqser_release::default::https://nexus.knecon.com/repository/red-platform-releases mvn --no-transfer-progress \ -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \ diff --git a/bamboo-specs/src/main/resources/scripts/sonar-java.sh b/bamboo-specs/src/main/resources/scripts/sonar-java.sh index a959a52..ba0b1de 100755 --- a/bamboo-specs/src/main/resources/scripts/sonar-java.sh +++ b/bamboo-specs/src/main/resources/scripts/sonar-java.sh @@ -22,7 +22,7 @@ then -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \ sonar:sonar \ -Dsonar.projectKey=RED_$SERVICE_NAME \ - -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.host.url=https://sonarqube.knecon.com \ -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ -Dsonar.branch.name=${bamboo_planRepository_1_branch} \ -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \ @@ -34,7 +34,7 @@ else -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \ sonar:sonar \ -Dsonar.projectKey=RED_$SERVICE_NAME \ - -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.host.url=https://sonarqube.knecon.com \ -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \ diff --git a/ocr-service-image-v1/pom.xml b/ocr-service-image-v1/pom.xml index fcf0ec3..95df898 100644 --- a/ocr-service-image-v1/pom.xml +++ b/ocr-service-image-v1/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - com.iqser.red + com.knecon.fforesight platform-docker-dependency - 1.2.0 + 0.1.0 4.0.0 From 961129b917e92b4dda2085215d3f15e205ec35a3 Mon Sep 17 00:00:00 2001 From: devplant Date: Thu, 25 May 2023 14:29:40 +0300 Subject: [PATCH 2/2] RED-6072 - As Operation I want to see why files are in an ERROR state - update the ocr failed with error information - in case of exception the reason is caught and added tot the message received from the queue --- ocr-service-v1/ocr-service-api-v1/pom.xml | 2 +- .../configuration/MessagingConfiguration.java | 2 + .../v1/server/service/OcrMessageReceiver.java | 53 ++++++++++++------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/ocr-service-v1/ocr-service-api-v1/pom.xml b/ocr-service-v1/ocr-service-api-v1/pom.xml index dc3c1a0..51c0c7b 100644 --- a/ocr-service-v1/ocr-service-api-v1/pom.xml +++ b/ocr-service-v1/ocr-service-api-v1/pom.xml @@ -13,7 +13,7 @@ ocr-service-api-v1 - 2.1.0 + 2.59.0 diff --git a/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/configuration/MessagingConfiguration.java b/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/configuration/MessagingConfiguration.java index 2c298dd..8f6d0ee 100644 --- a/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/configuration/MessagingConfiguration.java +++ b/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/configuration/MessagingConfiguration.java @@ -20,6 +20,8 @@ public class MessagingConfiguration { public static final String OCR_STATUS_UPDATE_RESPONSE_QUEUE = "ocr_status_update_response_queue"; + public static final String X_ERROR_INFO_HEADER = "x-error-message"; + @Bean public Queue ocrQueue() { diff --git a/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OcrMessageReceiver.java b/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OcrMessageReceiver.java index 65c2033..abb83b8 100644 --- a/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OcrMessageReceiver.java +++ b/ocr-service-v1/ocr-service-server-v1/src/main/java/com/iqser/red/service/ocr/v1/server/service/OcrMessageReceiver.java @@ -1,14 +1,19 @@ package com.iqser.red.service.ocr.v1.server.service; +import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.ocr.v1.api.model.DocumentRequest; import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; + import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.amqp.AmqpRejectAndDontRequeueException; +import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.annotation.RabbitHandler; import org.springframework.amqp.rabbit.annotation.RabbitListener; import org.springframework.http.HttpStatus; @@ -17,6 +22,7 @@ import org.springframework.stereotype.Service; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; @Slf4j @Service @@ -32,40 +38,47 @@ public class OcrMessageReceiver { @RabbitHandler @RabbitListener(queues = MessagingConfiguration.OCR_QUEUE, concurrency = "1") - public void receiveOcr(String in) throws JsonProcessingException { + public void receiveOcr(Message in) throws JsonProcessingException { - DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class); + DocumentRequest ocrRequestMessage = objectMapper.readValue(new String(in.getBody(), StandardCharsets.UTF_8), DocumentRequest.class); log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); - setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); + try { + setStatusOcrProcessing(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); - if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) { - byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); - fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile); - } - - try (var transferStream = new ByteArrayOutputStream()) { - ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream); - try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) { - fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream); + if (!fileStorageService.untouchedFileExists(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId())) { + byte[] originalFile = fileStorageService.getOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); + fileStorageService.storeUntouchedFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), originalFile); } - } catch (IOException e) { - log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); - throw new RuntimeException(e); - } - fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); + try (var transferStream = new ByteArrayOutputStream()) { + ocrService.runOcrOnDocument(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), transferStream); + try (var inputStream = new ByteArrayInputStream(transferStream.toByteArray())) { + fileStorageService.storeOriginalFile(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), inputStream); + } + } catch (IOException e) { + log.error("Failed to store file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); + throw new RuntimeException(e); + } + + fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); + } catch (Exception e) { + log.warn("An exception occurred in ocr file stage: ", e.getMessage() ); + in.getMessageProperties().getHeaders().put(X_ERROR_INFO_HEADER, e.getMessage()); + throw e; + } } @RabbitHandler @RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1") - public void receiveOcrDQL(String in) throws JsonProcessingException { + public void receiveOcrDQL(Message failedMessage) throws JsonProcessingException { - DocumentRequest ocrRequestMessage = objectMapper.readValue(in, DocumentRequest.class); + DocumentRequest ocrRequestMessage = objectMapper.readValue(new String(failedMessage.getBody(), StandardCharsets.UTF_8), DocumentRequest.class); log.info("OCR DQL received: {}", ocrRequestMessage); - fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId()); + String errorMessage = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER); + fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId(), new FileErrorInfo(errorMessage, MessagingConfiguration.OCR_DLQ, "ocr-service")); }