From 6110dd1497052ad0ca6b00c3115883ce79cc19e4 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 13 May 2024 15:06:25 +0200 Subject: [PATCH 01/15] RED-3813: image recategorization added first simple clustering service --- .../build.gradle.kts | 1 + .../service/image/Classification.java | 20 ++++++ .../service/image/FilterGeometry.java | 17 ++++++ .../v1/processor/service/image/Filters.java | 18 ++++++ .../v1/processor/service/image/Geometry.java | 17 ++++++ .../v1/processor/service/image/Image.java | 25 ++++++++ .../service/image/ImageClusteringService.java | 61 +++++++++++++++++++ .../processor/service/image/ImageFormat.java | 18 ++++++ .../service/image/ImageMetadata.java | 21 +++++++ .../service/image/ImageServiceResponse.java | 34 +++++++++++ .../v1/processor/service/image/ImageSize.java | 18 ++++++ .../v1/processor/service/image/Position.java | 20 ++++++ .../processor/service/image/Probability.java | 16 +++++ .../service/queue/ImageMessageReceiver.java | 10 ++- .../service/ImageClusteringTest.java | 5 ++ 15 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index ced472f2b..b564912f0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { api("commons-validator:commons-validator:1.7") implementation("org.mapstruct:mapstruct:1.5.5.Final") + implementation("org.apache.commons:commons-math3:3.6.1") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java new file mode 100644 index 000000000..42ac1de40 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Classification { + + private Map probabilities = new HashMap<>(); + private String label; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java new file mode 100644 index 000000000..7a8f248c8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FilterGeometry { + + private ImageSize imageSize; + private ImageFormat imageFormat; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java new file mode 100644 index 000000000..7c31d6b07 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Filters { + + private FilterGeometry geometry; + private Probability probability; + private boolean allPassed; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java new file mode 100644 index 000000000..78ac75668 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Geometry { + + private float width; + private float height; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java new file mode 100644 index 000000000..c1a8ab78e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import org.apache.commons.math3.ml.clustering.Clusterable; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +public class Image implements Clusterable { + private String hash; + private int fixedLength; + + @Override + public double[] getPoint() { + double[] featureVector = new double[fixedLength]; + for (int i = 0; i < hash.length(); i++) { + featureVector[i] = (double) hash.charAt(i); + } + return featureVector; + } +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java new file mode 100644 index 000000000..62bd9ed33 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.math3.ml.clustering.Cluster; +import org.apache.commons.math3.ml.clustering.DBSCANClusterer; +import org.apache.commons.math3.ml.distance.DistanceMeasure; +import org.apache.commons.math3.ml.distance.ManhattanDistance; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageClusteringService { + + private final ObjectMapper objectMapper; + private final double eps = 26; + private final int minPts = 3; + + private final int fixedLength = 25; + private final DistanceMeasure distanceMeasure = new ManhattanDistance(); + + @Autowired + FileManagementStorageService fileManagementStorageService; + + + public void clusterImages(String storageId) throws Exception { + + DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); + + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); + List imageMetadataList = imageServiceResponse.getData(); + List imageList = new ArrayList<>(); + for (ImageMetadata metadata : imageMetadataList) { + Image image = new Image(metadata.getRepresentation(), fixedLength); + imageList.add(image); + } + if (imageList.isEmpty()) { + throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); + } + List> clusters = dbscanClusterer.cluster(imageList); + for (Cluster cluster : clusters) { + List clusterImages = cluster.getPoints(); + log.info("cluster: {}", clusterImages.get(0)); + } + } + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java new file mode 100644 index 000000000..89bd37ad2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageFormat { + + private float quotient; + private boolean tooTall; + private boolean tooWide; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java new file mode 100644 index 000000000..5e7c59d50 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageMetadata { + + private Classification classification; + private String representation; + private Position position; + private Geometry geometry; + private Filters filters; + private boolean alpha; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java new file mode 100644 index 000000000..e2e548002 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageServiceResponse { + + private String dossierId; + private String fileId; + + @JsonProperty(value = "imageMetadata") + @JsonAlias("data") + private List data = new ArrayList<>(); + + private List dataCV = new ArrayList<>(); + + + @JsonProperty(value = "imageMetadata") + @JsonAlias("data") + public void setData(List data) {this.data = data;} + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java new file mode 100644 index 000000000..40343e0e3 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageSize { + + private float quotient; + private boolean tooLarge; + private boolean tooSmall; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java new file mode 100644 index 000000000..31b347f0e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Position { + + private float x1; + private float x2; + private float y1; + private float y2; + private int pageNumber; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java new file mode 100644 index 000000000..daec73d1d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Probability { + + private boolean unconfident; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 00597e9b2..8c3c31728 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,5 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; +import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -15,7 +16,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageClusteringService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -33,6 +36,7 @@ public class ImageMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final ObservationRegistry observationRegistry; + private final ImageClusteringService imageClusteringService; @SneakyThrows @@ -41,8 +45,13 @@ public class ImageMessageReceiver { JsonNode imageResponse = objectMapper.readTree(message.getBody()); + log.info("image response"+imageResponse); + String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); + imageClusteringService.clusterImages(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); + + addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -59,7 +68,6 @@ public class ImageMessageReceiver { String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); addFileIdToTrace(fileId); - log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java new file mode 100644 index 000000000..dad72ea06 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +public class ImageClusteringTest { + +} -- 2.47.2 From 4f1c926a172cac410e86da5f8a13683b671a3b04 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 13 May 2024 15:06:25 +0200 Subject: [PATCH 02/15] RED-3813: image recategorization added first simple clustering service --- .../build.gradle.kts | 1 + .../service/image/Classification.java | 20 ++++++ .../service/image/FilterGeometry.java | 17 ++++++ .../v1/processor/service/image/Filters.java | 18 ++++++ .../v1/processor/service/image/Geometry.java | 17 ++++++ .../v1/processor/service/image/Image.java | 25 ++++++++ .../service/image/ImageClusteringService.java | 61 +++++++++++++++++++ .../processor/service/image/ImageFormat.java | 18 ++++++ .../service/image/ImageMetadata.java | 21 +++++++ .../service/image/ImageServiceResponse.java | 34 +++++++++++ .../v1/processor/service/image/ImageSize.java | 18 ++++++ .../v1/processor/service/image/Position.java | 20 ++++++ .../processor/service/image/Probability.java | 16 +++++ .../service/queue/ImageMessageReceiver.java | 10 ++- .../service/ImageClusteringTest.java | 5 ++ 15 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index ced472f2b..b564912f0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -61,6 +61,7 @@ dependencies { api("commons-validator:commons-validator:1.7") implementation("org.mapstruct:mapstruct:1.5.5.Final") + implementation("org.apache.commons:commons-math3:3.6.1") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java new file mode 100644 index 000000000..42ac1de40 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Classification { + + private Map probabilities = new HashMap<>(); + private String label; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java new file mode 100644 index 000000000..7a8f248c8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FilterGeometry { + + private ImageSize imageSize; + private ImageFormat imageFormat; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java new file mode 100644 index 000000000..7c31d6b07 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Filters { + + private FilterGeometry geometry; + private Probability probability; + private boolean allPassed; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java new file mode 100644 index 000000000..78ac75668 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Geometry { + + private float width; + private float height; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java new file mode 100644 index 000000000..c1a8ab78e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import org.apache.commons.math3.ml.clustering.Clusterable; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +public class Image implements Clusterable { + private String hash; + private int fixedLength; + + @Override + public double[] getPoint() { + double[] featureVector = new double[fixedLength]; + for (int i = 0; i < hash.length(); i++) { + featureVector[i] = (double) hash.charAt(i); + } + return featureVector; + } +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java new file mode 100644 index 000000000..62bd9ed33 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.math3.ml.clustering.Cluster; +import org.apache.commons.math3.ml.clustering.DBSCANClusterer; +import org.apache.commons.math3.ml.distance.DistanceMeasure; +import org.apache.commons.math3.ml.distance.ManhattanDistance; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageClusteringService { + + private final ObjectMapper objectMapper; + private final double eps = 26; + private final int minPts = 3; + + private final int fixedLength = 25; + private final DistanceMeasure distanceMeasure = new ManhattanDistance(); + + @Autowired + FileManagementStorageService fileManagementStorageService; + + + public void clusterImages(String storageId) throws Exception { + + DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); + + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); + List imageMetadataList = imageServiceResponse.getData(); + List imageList = new ArrayList<>(); + for (ImageMetadata metadata : imageMetadataList) { + Image image = new Image(metadata.getRepresentation(), fixedLength); + imageList.add(image); + } + if (imageList.isEmpty()) { + throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); + } + List> clusters = dbscanClusterer.cluster(imageList); + for (Cluster cluster : clusters) { + List clusterImages = cluster.getPoints(); + log.info("cluster: {}", clusterImages.get(0)); + } + } + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java new file mode 100644 index 000000000..89bd37ad2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageFormat { + + private float quotient; + private boolean tooTall; + private boolean tooWide; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java new file mode 100644 index 000000000..5e7c59d50 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageMetadata { + + private Classification classification; + private String representation; + private Position position; + private Geometry geometry; + private Filters filters; + private boolean alpha; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java new file mode 100644 index 000000000..e2e548002 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageServiceResponse { + + private String dossierId; + private String fileId; + + @JsonProperty(value = "imageMetadata") + @JsonAlias("data") + private List data = new ArrayList<>(); + + private List dataCV = new ArrayList<>(); + + + @JsonProperty(value = "imageMetadata") + @JsonAlias("data") + public void setData(List data) {this.data = data;} + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java new file mode 100644 index 000000000..40343e0e3 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageSize { + + private float quotient; + private boolean tooLarge; + private boolean tooSmall; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java new file mode 100644 index 000000000..31b347f0e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Position { + + private float x1; + private float x2; + private float y1; + private float y2; + private int pageNumber; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java new file mode 100644 index 000000000..daec73d1d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Probability { + + private boolean unconfident; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 00597e9b2..8c3c31728 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,5 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; +import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -15,7 +16,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageClusteringService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -33,6 +36,7 @@ public class ImageMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final ObservationRegistry observationRegistry; + private final ImageClusteringService imageClusteringService; @SneakyThrows @@ -41,8 +45,13 @@ public class ImageMessageReceiver { JsonNode imageResponse = objectMapper.readTree(message.getBody()); + log.info("image response"+imageResponse); + String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); + imageClusteringService.clusterImages(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); + + addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -59,7 +68,6 @@ public class ImageMessageReceiver { String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); addFileIdToTrace(fileId); - log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java new file mode 100644 index 000000000..dad72ea06 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +public class ImageClusteringTest { + +} -- 2.47.2 From a4426e87ce14a049acc5339967ed6afb69ed2958 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:03:13 +0200 Subject: [PATCH 03/15] RED-3813: image recategorization added saving images after layout parsing finishes added endpoint for querying similiar images --- .../ImageSimilaritySearchController.java | 51 +++++++++ .../ImageSimilaritySearchResource.java | 28 +++++ .../v1/processor/roles/ActionRoles.java | 4 + .../v1/processor/service/image/Image.java | 1 + .../service/image/ImageClusteringService.java | 61 ----------- .../service/image/ImageSimilarityService.java | 96 ++++++++++++++++ .../service/queue/ImageMessageReceiver.java | 11 +- .../LayoutParsingFinishedMessageReceiver.java | 30 +++-- .../image/ImageSimilaritySearchRequest.java | 31 ++++++ .../image/ImageSimilaritySearchResponse.java | 28 +++++ .../v1/api/shared/model/utils/Scope.java | 14 +++ .../shared/mongo/document/ImageDocument.java | 40 +++++++ .../repository/ImageDocumentRepository.java | 12 ++ .../mongo/service/ImageMongoService.java | 103 ++++++++++++++++++ 14 files changed, 428 insertions(+), 82 deletions(-) create mode 100644 persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java create mode 100644 persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java create mode 100644 persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java create mode 100644 persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java create mode 100644 persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java new file mode 100644 index 000000000..23839b430 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java @@ -0,0 +1,51 @@ +package com.iqser.red.persistence.service.v1.external.api.impl.controller; + +import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_SIMILIAR_IMAGES; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; +import com.iqser.red.service.persistence.service.v1.api.external.resource.ImageSimilaritySearchResource; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@RestController +@RequiredArgsConstructor +@Slf4j +public class ImageSimilaritySearchController implements ImageSimilaritySearchResource { + + private final ImageSimilarityService imageSimiliarityService; + + + @SneakyThrows + public ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest) { + + + log.info("received similiar image search request {}",imageSimilaritySearchRequest); + List similarImages = this.imageSimiliarityService.findSimilarImages(imageSimilaritySearchRequest.getCentroId(), + imageSimilaritySearchRequest.getDistance(), + imageSimilaritySearchRequest.getScope()); + List similarImagesIds = new ArrayList<>(); + List similarImagesThumbnails = new ArrayList<>(); + similarImages.stream() + .forEach(doc -> { + similarImagesIds.add(doc.getImageId()); + similarImagesThumbnails.add(doc.getThumbnail()); + }); + ImageSimilaritySearchResponse imageSimilaritySearchResponse = new ImageSimilaritySearchResponse(similarImagesIds, similarImagesThumbnails); + return new ResponseEntity<>(imageSimilaritySearchResponse, HttpStatus.OK); + } + +} diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java new file mode 100644 index 000000000..a8795cdf3 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.service.v1.api.external.resource; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchResponse; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; + +public interface ImageSimilaritySearchResource { + + String IMAGE_SIMILARITY_SEARCH_PATH = ExternalApi.BASE_PATH + "/imageSimilaritySearch"; + + + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Gets similiar images to given image", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")}) + @PostMapping(value = IMAGE_SIMILARITY_SEARCH_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java index 73b62c34f..9472d1111 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java @@ -62,6 +62,10 @@ public final class ActionRoles { public static final String READ_GENERAL_CONFIGURATION = "red-read-general-configuration"; public static final String WRITE_GENERAL_CONFIGURATION = "red-write-general-configuration"; + // IMAGE SIMILIARITY SEARCH + + public static final String GET_SIMILIAR_IMAGES ="red-get-similiar-images"; + // Preferences public static final String MANAGE_USER_PREFERENCES = "red-manage-user-preferences"; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java index c1a8ab78e..ca807331c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java @@ -13,6 +13,7 @@ import lombok.NoArgsConstructor; public class Image implements Clusterable { private String hash; private int fixedLength; + private String label; @Override public double[] getPoint() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java deleted file mode 100644 index 62bd9ed33..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.math3.ml.clustering.Cluster; -import org.apache.commons.math3.ml.clustering.DBSCANClusterer; -import org.apache.commons.math3.ml.distance.DistanceMeasure; -import org.apache.commons.math3.ml.distance.ManhattanDistance; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.knecon.fforesight.tenantcommons.TenantContext; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class ImageClusteringService { - - private final ObjectMapper objectMapper; - private final double eps = 26; - private final int minPts = 3; - - private final int fixedLength = 25; - private final DistanceMeasure distanceMeasure = new ManhattanDistance(); - - @Autowired - FileManagementStorageService fileManagementStorageService; - - - public void clusterImages(String storageId) throws Exception { - - DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); - - try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { - ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); - List imageMetadataList = imageServiceResponse.getData(); - List imageList = new ArrayList<>(); - for (ImageMetadata metadata : imageMetadataList) { - Image image = new Image(metadata.getRepresentation(), fixedLength); - imageList.add(image); - } - if (imageList.isEmpty()) { - throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); - } - List> clusters = dbscanClusterer.cluster(imageList); - for (Cluster cluster : clusters) { - List clusterImages = cluster.getPoints(); - log.info("cluster: {}", clusterImages.get(0)); - } - } - - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java new file mode 100644 index 000000000..f9111a584 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java @@ -0,0 +1,96 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class ImageSimilarityService { + + @Autowired + ImageMongoService imageMongoService; + @Autowired + FileManagementStorageService fileManagementStorageService; + @Autowired + ObjectMapper objectMapper; + + + //load all recat requests and find similiar images + public void saveImages(String templateId, String dossierId, String fileId, String storageId) throws IOException { + //load structure files of all files in template + List imageDocuments = new ArrayList<>(); + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + //load images from structures + DocumentStructure documentStructure = objectMapper.readValue(inputStream, DocumentStructure.class); + documentStructure.streamAllEntries() + .filter(entry -> entry.getType().equals(NodeType.IMAGE)) + .forEach(i -> { + Map properties = i.getProperties(); + ImageDocument imageDocument = new ImageDocument(); + imageDocument.setImageId(properties.get("id")); + imageDocument.setFeatureVector(parseRepresentationVector(properties.get("representationHash"))); + imageDocument.setTemplateId(templateId); + imageDocument.setDossierId(dossierId); + imageDocument.setFileId(fileId); + imageDocuments.add(imageDocument); + + }); + } + if (imageDocuments.isEmpty()) { + return; + } + imageMongoService.saveImages(imageDocuments); + } + + + public List findSimilarImages(String centroId, double distance, Scope scope) throws Exception { + + ImageDocument centroImage = this.imageMongoService.findById(centroId); + log.info("image received with id {}: {}",centroId, centroImage); + List similarImages = this.imageMongoService.findSimilarImages(centroImage, distance, scope); + log.info("received similar images: {}",similarImages); + return similarImages.stream() + .collect(Collectors.toList()); + } + + + public static double[] parseRepresentationVector(String representationHash) { + + double[] doubleArray = new double[representationHash.length()]; + + for (int i = 0; i < representationHash.length(); i++) { + char c = representationHash.charAt(i); + if (Character.isDigit(c)) { + // Convert numeric characters directly to their numeric values + doubleArray[i] = Character.getNumericValue(c); + } else if (Character.isLetter(c)) { + // Convert alphabetic characters to their position in the alphabet + // 'A' or 'a' -> 10, 'B' or 'b' -> 11, ..., 'F' or 'f' -> 15 + doubleArray[i] = 10 + Character.toUpperCase(c) - 'A'; + } else { + throw new IllegalArgumentException("Invalid character in input string: " + c); + } + } + + return doubleArray; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 8c3c31728..64ba4d923 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; -import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -16,11 +15,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageClusteringService; -import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; -import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; @@ -36,7 +31,6 @@ public class ImageMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final ObservationRegistry observationRegistry; - private final ImageClusteringService imageClusteringService; @SneakyThrows @@ -45,18 +39,15 @@ public class ImageMessageReceiver { JsonNode imageResponse = objectMapper.readTree(message.getBody()); - log.info("image response"+imageResponse); - String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); - imageClusteringService.clusterImages(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); - addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); log.info("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_RESPONSE_QUEUE, dossierId, fileId); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 5b6003e96..49a7762d7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -1,14 +1,23 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.exc.ValueInstantiationException; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; @@ -17,13 +26,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.core.Message; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; - @Slf4j @Service @RequiredArgsConstructor @@ -35,6 +37,7 @@ public class LayoutParsingFinishedMessageReceiver { private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; private final SaasMigrationService saasMigrationService; + private final ImageSimilarityService imageSimiliarityService; @SneakyThrows @@ -47,11 +50,16 @@ public class LayoutParsingFinishedMessageReceiver { return; } - fileStatusService.setStatusAnalyse(layoutParsingRequestIdentifierService.parseDossierId(response.identifier()), - layoutParsingRequestIdentifierService.parseFileId(response.identifier()), - layoutParsingRequestIdentifierService.parsePriority(response.identifier())); + String templateId = ""; + String dossierId = layoutParsingRequestIdentifierService.parseDossierId(response.identifier()); + String fileId = layoutParsingRequestIdentifierService.parseFileId(response.identifier()); + String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); + + + fileStatusService.setStatusAnalyse(dossierId, fileId, layoutParsingRequestIdentifierService.parsePriority(response.identifier())); fileStatusService.updateLayoutProcessedTime(layoutParsingRequestIdentifierService.parseFileId(response.identifier())); + imageSimiliarityService.saveImages(templateId, dossierId, fileId, storageId); log.info("Received message {} in {}", response, LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE); } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java new file mode 100644 index 000000000..418d51d09 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.image; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.ToString; + +@Data +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +@Schema(description = "Image Similiarity Search request containing information about centroId, distance and scope") +public class ImageSimilaritySearchRequest { + + @NonNull + @Schema(description = "id of the central image") + private String centroId; + @NonNull + @Schema(description = "the manhattan distance used in the similiarity search") + private double distance; + @NonNull + @Schema(description = "the scope for the similiarity search") + private Scope scope; + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java new file mode 100644 index 000000000..03db80643 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.image; + +import java.util.ArrayList; +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +@Schema(description = "Object containing the result of the image similiarity search") +public class ImageSimilaritySearchResponse { + + @Builder.Default + @Schema(description = "list of ids of similiar images") + List similarImagesIds = new ArrayList<>(); + + @Builder.Default + @Schema(description = "list of ids of similiar images") + List similarImagesThumbnails = new ArrayList<>(); + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java new file mode 100644 index 000000000..f9a881f19 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java @@ -0,0 +1,14 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.utils; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class Scope { + private String templateId; + private String dossierId; + private String fileId; +} diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java new file mode 100644 index 000000000..609226afc --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java @@ -0,0 +1,40 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.mongo.document; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@Document(collection = "images") +public class ImageDocument { + + @Indexed + private double[] featureVector; + + private String label; + + @Id + @EqualsAndHashCode.Include + private String imageId; + + private String templateId; + + private String dossierId; + + private String fileId; + + private Byte thumbnail; + +} diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java new file mode 100644 index 000000000..1dbde7d6f --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java @@ -0,0 +1,12 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; + +@Repository +public interface ImageDocumentRepository extends MongoRepository { + // Additional query methods can be added here + +} diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java new file mode 100644 index 000000000..29517993c --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java @@ -0,0 +1,103 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.mongo.service; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.data.mongodb.core.MongoOperations; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.ImageDocumentRepository; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class ImageMongoService { + + private MongoOperations mongoOperations; + private final ImageDocumentRepository imageDocumentRepository; + + + public ImageMongoService(MongoOperations mongoOperations, ImageDocumentRepository imageDocumentRepository) { + + this.mongoOperations = mongoOperations; + + this.imageDocumentRepository = imageDocumentRepository; + } + + + public ImageDocument findById(String imageId) { + + Optional image = this.imageDocumentRepository.findById(imageId); + if (image.isPresent()) { + return image.get(); + } + return null; + } + + + public List findSimilarImages(ImageDocument centralImage, double maxDistance, Scope scope) { + + double[] centralFeatureVector = centralImage.getFeatureVector(); + + List allImages = this.imageDocumentRepository.findAll(); + + return allImages.stream() + .filter(image -> !image.getImageId().equals(centralImage.getImageId())) + .filter(image -> filterForScope(image, scope)) + .map(image -> new ImageDistancePair(image, calculateManhattanDistance(image.getFeatureVector(), centralFeatureVector))) + .filter(image -> image.getDistance() <= maxDistance) + .map(ImageDistancePair::getImageDocument) + .collect(Collectors.toList()); + } + + + private boolean filterForScope(ImageDocument image, Scope scope) { + + if (scope.getTemplateId() != null && scope.getTemplateId().length() > 0) { + return image.getTemplateId() != null && image.getTemplateId().equals(scope.getTemplateId()); + } else if (scope.getDossierId() != null && scope.getDossierId().length() > 0) { + return image.getDossierId() != null && image.getDossierId().equals(scope.getDossierId()); + } else if (scope.getFileId() != null && scope.getFileId().length() > 0) { + return image.getFileId() != null && image.getFileId().equals(scope.getFileId()); + } + return true; + } + + + private double calculateManhattanDistance(double[] vector1, double[] vector2) { + + double distance = 0.0; + for (int i = 0; i < vector1.length; i++) { + distance += Math.abs(vector1[i] - vector2[i]); + } + return distance; + } + + + public void saveImages(List imageDocuments) { + + imageDocumentRepository.saveAll(imageDocuments); + } + + + @Getter + private static class ImageDistancePair { + + private final ImageDocument imageDocument; + private final double distance; + + + public ImageDistancePair(ImageDocument imageDocument, double distance) { + + this.imageDocument = imageDocument; + this.distance = distance; + } + + } + +} -- 2.47.2 From 9c48547c360997472cafab06a0518c4cabbc4533 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 13 May 2024 15:06:25 +0200 Subject: [PATCH 04/15] RED-3813: image recategorization added first simple clustering service --- .../build.gradle.kts | 1 + .../service/image/Classification.java | 20 ++++++ .../service/image/FilterGeometry.java | 17 ++++++ .../v1/processor/service/image/Filters.java | 18 ++++++ .../v1/processor/service/image/Geometry.java | 17 ++++++ .../v1/processor/service/image/Image.java | 25 ++++++++ .../service/image/ImageClusteringService.java | 61 +++++++++++++++++++ .../processor/service/image/ImageFormat.java | 18 ++++++ .../service/image/ImageMetadata.java | 21 +++++++ .../service/image/ImageServiceResponse.java | 34 +++++++++++ .../v1/processor/service/image/ImageSize.java | 18 ++++++ .../v1/processor/service/image/Position.java | 20 ++++++ .../processor/service/image/Probability.java | 16 +++++ .../service/queue/ImageMessageReceiver.java | 10 ++- .../service/ImageClusteringTest.java | 5 ++ 15 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 05f998c98..634002c14 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -63,6 +63,7 @@ dependencies { api("com.opencsv:opencsv:5.9") implementation("org.mapstruct:mapstruct:1.5.5.Final") + implementation("org.apache.commons:commons-math3:3.6.1") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java new file mode 100644 index 000000000..42ac1de40 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Classification { + + private Map probabilities = new HashMap<>(); + private String label; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java new file mode 100644 index 000000000..7a8f248c8 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FilterGeometry { + + private ImageSize imageSize; + private ImageFormat imageFormat; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java new file mode 100644 index 000000000..7c31d6b07 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Filters { + + private FilterGeometry geometry; + private Probability probability; + private boolean allPassed; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java new file mode 100644 index 000000000..78ac75668 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Geometry { + + private float width; + private float height; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java new file mode 100644 index 000000000..c1a8ab78e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import org.apache.commons.math3.ml.clustering.Clusterable; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +public class Image implements Clusterable { + private String hash; + private int fixedLength; + + @Override + public double[] getPoint() { + double[] featureVector = new double[fixedLength]; + for (int i = 0; i < hash.length(); i++) { + featureVector[i] = (double) hash.charAt(i); + } + return featureVector; + } +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java new file mode 100644 index 000000000..62bd9ed33 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.math3.ml.clustering.Cluster; +import org.apache.commons.math3.ml.clustering.DBSCANClusterer; +import org.apache.commons.math3.ml.distance.DistanceMeasure; +import org.apache.commons.math3.ml.distance.ManhattanDistance; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageClusteringService { + + private final ObjectMapper objectMapper; + private final double eps = 26; + private final int minPts = 3; + + private final int fixedLength = 25; + private final DistanceMeasure distanceMeasure = new ManhattanDistance(); + + @Autowired + FileManagementStorageService fileManagementStorageService; + + + public void clusterImages(String storageId) throws Exception { + + DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); + + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); + List imageMetadataList = imageServiceResponse.getData(); + List imageList = new ArrayList<>(); + for (ImageMetadata metadata : imageMetadataList) { + Image image = new Image(metadata.getRepresentation(), fixedLength); + imageList.add(image); + } + if (imageList.isEmpty()) { + throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); + } + List> clusters = dbscanClusterer.cluster(imageList); + for (Cluster cluster : clusters) { + List clusterImages = cluster.getPoints(); + log.info("cluster: {}", clusterImages.get(0)); + } + } + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java new file mode 100644 index 000000000..89bd37ad2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageFormat { + + private float quotient; + private boolean tooTall; + private boolean tooWide; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java new file mode 100644 index 000000000..5e7c59d50 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageMetadata { + + private Classification classification; + private String representation; + private Position position; + private Geometry geometry; + private Filters filters; + private boolean alpha; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java new file mode 100644 index 000000000..e2e548002 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageServiceResponse { + + private String dossierId; + private String fileId; + + @JsonProperty(value = "imageMetadata") + @JsonAlias("data") + private List data = new ArrayList<>(); + + private List dataCV = new ArrayList<>(); + + + @JsonProperty(value = "imageMetadata") + @JsonAlias("data") + public void setData(List data) {this.data = data;} + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java new file mode 100644 index 000000000..40343e0e3 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImageSize { + + private float quotient; + private boolean tooLarge; + private boolean tooSmall; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java new file mode 100644 index 000000000..31b347f0e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Position { + + private float x1; + private float x2; + private float y1; + private float y2; + private int pageNumber; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java new file mode 100644 index 000000000..daec73d1d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Probability { + + private boolean unconfident; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 00597e9b2..8c3c31728 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,5 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; +import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -15,7 +16,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageClusteringService; import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -33,6 +36,7 @@ public class ImageMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final ObservationRegistry observationRegistry; + private final ImageClusteringService imageClusteringService; @SneakyThrows @@ -41,8 +45,13 @@ public class ImageMessageReceiver { JsonNode imageResponse = objectMapper.readTree(message.getBody()); + log.info("image response"+imageResponse); + String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); + imageClusteringService.clusterImages(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); + + addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); @@ -59,7 +68,6 @@ public class ImageMessageReceiver { String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); addFileIdToTrace(fileId); - log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java new file mode 100644 index 000000000..dad72ea06 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +public class ImageClusteringTest { + +} -- 2.47.2 From 23b437e60ac3c86100a3634201d39ec3e04a3134 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:03:13 +0200 Subject: [PATCH 05/15] RED-3813: image recategorization added saving images after layout parsing finishes added endpoint for querying similiar images --- .../ImageSimilaritySearchController.java | 51 +++++++++ .../ImageSimilaritySearchResource.java | 28 +++++ .../v1/processor/roles/ActionRoles.java | 4 + .../v1/processor/service/image/Image.java | 1 + .../service/image/ImageClusteringService.java | 61 ----------- .../service/image/ImageSimilarityService.java | 96 ++++++++++++++++ .../service/queue/ImageMessageReceiver.java | 11 +- .../LayoutParsingFinishedMessageReceiver.java | 10 +- .../image/ImageSimilaritySearchRequest.java | 31 ++++++ .../image/ImageSimilaritySearchResponse.java | 28 +++++ .../v1/api/shared/model/utils/Scope.java | 14 +++ .../shared/mongo/document/ImageDocument.java | 40 +++++++ .../repository/ImageDocumentRepository.java | 12 ++ .../mongo/service/ImageMongoService.java | 103 ++++++++++++++++++ 14 files changed, 416 insertions(+), 74 deletions(-) create mode 100644 persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java create mode 100644 persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java create mode 100644 persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java create mode 100644 persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java create mode 100644 persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java new file mode 100644 index 000000000..23839b430 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java @@ -0,0 +1,51 @@ +package com.iqser.red.persistence.service.v1.external.api.impl.controller; + +import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_SIMILIAR_IMAGES; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; +import com.iqser.red.service.persistence.service.v1.api.external.resource.ImageSimilaritySearchResource; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchResponse; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@RestController +@RequiredArgsConstructor +@Slf4j +public class ImageSimilaritySearchController implements ImageSimilaritySearchResource { + + private final ImageSimilarityService imageSimiliarityService; + + + @SneakyThrows + public ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest) { + + + log.info("received similiar image search request {}",imageSimilaritySearchRequest); + List similarImages = this.imageSimiliarityService.findSimilarImages(imageSimilaritySearchRequest.getCentroId(), + imageSimilaritySearchRequest.getDistance(), + imageSimilaritySearchRequest.getScope()); + List similarImagesIds = new ArrayList<>(); + List similarImagesThumbnails = new ArrayList<>(); + similarImages.stream() + .forEach(doc -> { + similarImagesIds.add(doc.getImageId()); + similarImagesThumbnails.add(doc.getThumbnail()); + }); + ImageSimilaritySearchResponse imageSimilaritySearchResponse = new ImageSimilaritySearchResponse(similarImagesIds, similarImagesThumbnails); + return new ResponseEntity<>(imageSimilaritySearchResponse, HttpStatus.OK); + } + +} diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java new file mode 100644 index 000000000..a8795cdf3 --- /dev/null +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/ImageSimilaritySearchResource.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.service.v1.api.external.resource; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchResponse; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; + +public interface ImageSimilaritySearchResource { + + String IMAGE_SIMILARITY_SEARCH_PATH = ExternalApi.BASE_PATH + "/imageSimilaritySearch"; + + + @ResponseStatus(value = HttpStatus.OK) + @Operation(summary = "Gets similiar images to given image", description = "None") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Not found")}) + @PostMapping(value = IMAGE_SIMILARITY_SEARCH_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java index 051962562..8113193a0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java @@ -59,6 +59,10 @@ public final class ActionRoles { public static final String READ_GENERAL_CONFIGURATION = "red-read-general-configuration"; public static final String WRITE_GENERAL_CONFIGURATION = "red-write-general-configuration"; + // IMAGE SIMILIARITY SEARCH + + public static final String GET_SIMILIAR_IMAGES ="red-get-similiar-images"; + // Preferences public static final String MANAGE_USER_PREFERENCES = "red-manage-user-preferences"; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java index c1a8ab78e..ca807331c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java @@ -13,6 +13,7 @@ import lombok.NoArgsConstructor; public class Image implements Clusterable { private String hash; private int fixedLength; + private String label; @Override public double[] getPoint() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java deleted file mode 100644 index 62bd9ed33..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.math3.ml.clustering.Cluster; -import org.apache.commons.math3.ml.clustering.DBSCANClusterer; -import org.apache.commons.math3.ml.distance.DistanceMeasure; -import org.apache.commons.math3.ml.distance.ManhattanDistance; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.knecon.fforesight.tenantcommons.TenantContext; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class ImageClusteringService { - - private final ObjectMapper objectMapper; - private final double eps = 26; - private final int minPts = 3; - - private final int fixedLength = 25; - private final DistanceMeasure distanceMeasure = new ManhattanDistance(); - - @Autowired - FileManagementStorageService fileManagementStorageService; - - - public void clusterImages(String storageId) throws Exception { - - DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); - - try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { - ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); - List imageMetadataList = imageServiceResponse.getData(); - List imageList = new ArrayList<>(); - for (ImageMetadata metadata : imageMetadataList) { - Image image = new Image(metadata.getRepresentation(), fixedLength); - imageList.add(image); - } - if (imageList.isEmpty()) { - throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); - } - List> clusters = dbscanClusterer.cluster(imageList); - for (Cluster cluster : clusters) { - List clusterImages = cluster.getPoints(); - log.info("cluster: {}", clusterImages.get(0)); - } - } - - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java new file mode 100644 index 000000000..f9111a584 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java @@ -0,0 +1,96 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class ImageSimilarityService { + + @Autowired + ImageMongoService imageMongoService; + @Autowired + FileManagementStorageService fileManagementStorageService; + @Autowired + ObjectMapper objectMapper; + + + //load all recat requests and find similiar images + public void saveImages(String templateId, String dossierId, String fileId, String storageId) throws IOException { + //load structure files of all files in template + List imageDocuments = new ArrayList<>(); + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + //load images from structures + DocumentStructure documentStructure = objectMapper.readValue(inputStream, DocumentStructure.class); + documentStructure.streamAllEntries() + .filter(entry -> entry.getType().equals(NodeType.IMAGE)) + .forEach(i -> { + Map properties = i.getProperties(); + ImageDocument imageDocument = new ImageDocument(); + imageDocument.setImageId(properties.get("id")); + imageDocument.setFeatureVector(parseRepresentationVector(properties.get("representationHash"))); + imageDocument.setTemplateId(templateId); + imageDocument.setDossierId(dossierId); + imageDocument.setFileId(fileId); + imageDocuments.add(imageDocument); + + }); + } + if (imageDocuments.isEmpty()) { + return; + } + imageMongoService.saveImages(imageDocuments); + } + + + public List findSimilarImages(String centroId, double distance, Scope scope) throws Exception { + + ImageDocument centroImage = this.imageMongoService.findById(centroId); + log.info("image received with id {}: {}",centroId, centroImage); + List similarImages = this.imageMongoService.findSimilarImages(centroImage, distance, scope); + log.info("received similar images: {}",similarImages); + return similarImages.stream() + .collect(Collectors.toList()); + } + + + public static double[] parseRepresentationVector(String representationHash) { + + double[] doubleArray = new double[representationHash.length()]; + + for (int i = 0; i < representationHash.length(); i++) { + char c = representationHash.charAt(i); + if (Character.isDigit(c)) { + // Convert numeric characters directly to their numeric values + doubleArray[i] = Character.getNumericValue(c); + } else if (Character.isLetter(c)) { + // Convert alphabetic characters to their position in the alphabet + // 'A' or 'a' -> 10, 'B' or 'b' -> 11, ..., 'F' or 'f' -> 15 + doubleArray[i] = 10 + Character.toUpperCase(c) - 'A'; + } else { + throw new IllegalArgumentException("Invalid character in input string: " + c); + } + } + + return doubleArray; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 8c3c31728..64ba4d923 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; -import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -16,11 +15,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageClusteringService; -import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings; -import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import io.micrometer.observation.ObservationRegistry; import lombok.RequiredArgsConstructor; @@ -36,7 +31,6 @@ public class ImageMessageReceiver { private final ObjectMapper objectMapper; private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService; private final ObservationRegistry observationRegistry; - private final ImageClusteringService imageClusteringService; @SneakyThrows @@ -45,18 +39,15 @@ public class ImageMessageReceiver { JsonNode imageResponse = objectMapper.readTree(message.getBody()); - log.info("image response"+imageResponse); - String dossierId = imageResponse.path("dossierId").asText(); String fileId = imageResponse.path("fileId").asText(); - imageClusteringService.clusterImages(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); - addFileIdToTrace(fileId); fileStatusService.setStatusAnalyse(dossierId, fileId, false); log.info("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_RESPONSE_QUEUE, dossierId, fileId); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 4ac94ca82..9a45adb74 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -4,10 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.ValueInstantiationException; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; -import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; @@ -37,7 +35,7 @@ public class LayoutParsingFinishedMessageReceiver { private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; private final SaasMigrationService saasMigrationService; - private final WebsocketService websocketService; + private final ImageSimilarityService imageSimiliarityService; @SneakyThrows @@ -50,6 +48,11 @@ public class LayoutParsingFinishedMessageReceiver { return; } + String templateId = ""; + String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); + + + fileStatusService.setStatusAnalyse(dossierId, fileId, layoutParsingRequestIdentifierService.parsePriority(response.identifier())); var dossierId = layoutParsingRequestIdentifierService.parseDossierId(response.identifier()); var fileId = layoutParsingRequestIdentifierService.parseFileId(response.identifier()); @@ -58,6 +61,7 @@ public class LayoutParsingFinishedMessageReceiver { layoutParsingRequestIdentifierService.parsePriority(response.identifier())); fileStatusService.updateLayoutProcessedTime(layoutParsingRequestIdentifierService.parseFileId(response.identifier())); + imageSimiliarityService.saveImages(templateId, dossierId, fileId, storageId); websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.LAYOUT_UPDATE, fileStatusService.getStatus(fileId).getNumberOfAnalyses() + 1); log.info("Received message {} in {}", response, LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java new file mode 100644 index 000000000..418d51d09 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchRequest.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.image; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.ToString; + +@Data +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +@Schema(description = "Image Similiarity Search request containing information about centroId, distance and scope") +public class ImageSimilaritySearchRequest { + + @NonNull + @Schema(description = "id of the central image") + private String centroId; + @NonNull + @Schema(description = "the manhattan distance used in the similiarity search") + private double distance; + @NonNull + @Schema(description = "the scope for the similiarity search") + private Scope scope; + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java new file mode 100644 index 000000000..03db80643 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/image/ImageSimilaritySearchResponse.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.image; + +import java.util.ArrayList; +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +@Schema(description = "Object containing the result of the image similiarity search") +public class ImageSimilaritySearchResponse { + + @Builder.Default + @Schema(description = "list of ids of similiar images") + List similarImagesIds = new ArrayList<>(); + + @Builder.Default + @Schema(description = "list of ids of similiar images") + List similarImagesThumbnails = new ArrayList<>(); + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java new file mode 100644 index 000000000..f9a881f19 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/Scope.java @@ -0,0 +1,14 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.utils; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class Scope { + private String templateId; + private String dossierId; + private String fileId; +} diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java new file mode 100644 index 000000000..609226afc --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/document/ImageDocument.java @@ -0,0 +1,40 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.mongo.document; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@ToString +@Document(collection = "images") +public class ImageDocument { + + @Indexed + private double[] featureVector; + + private String label; + + @Id + @EqualsAndHashCode.Include + private String imageId; + + private String templateId; + + private String dossierId; + + private String fileId; + + private Byte thumbnail; + +} diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java new file mode 100644 index 000000000..1dbde7d6f --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/ImageDocumentRepository.java @@ -0,0 +1,12 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; + +@Repository +public interface ImageDocumentRepository extends MongoRepository { + // Additional query methods can be added here + +} diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java new file mode 100644 index 000000000..29517993c --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java @@ -0,0 +1,103 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.mongo.service; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.data.mongodb.core.MongoOperations; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.ImageDocumentRepository; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class ImageMongoService { + + private MongoOperations mongoOperations; + private final ImageDocumentRepository imageDocumentRepository; + + + public ImageMongoService(MongoOperations mongoOperations, ImageDocumentRepository imageDocumentRepository) { + + this.mongoOperations = mongoOperations; + + this.imageDocumentRepository = imageDocumentRepository; + } + + + public ImageDocument findById(String imageId) { + + Optional image = this.imageDocumentRepository.findById(imageId); + if (image.isPresent()) { + return image.get(); + } + return null; + } + + + public List findSimilarImages(ImageDocument centralImage, double maxDistance, Scope scope) { + + double[] centralFeatureVector = centralImage.getFeatureVector(); + + List allImages = this.imageDocumentRepository.findAll(); + + return allImages.stream() + .filter(image -> !image.getImageId().equals(centralImage.getImageId())) + .filter(image -> filterForScope(image, scope)) + .map(image -> new ImageDistancePair(image, calculateManhattanDistance(image.getFeatureVector(), centralFeatureVector))) + .filter(image -> image.getDistance() <= maxDistance) + .map(ImageDistancePair::getImageDocument) + .collect(Collectors.toList()); + } + + + private boolean filterForScope(ImageDocument image, Scope scope) { + + if (scope.getTemplateId() != null && scope.getTemplateId().length() > 0) { + return image.getTemplateId() != null && image.getTemplateId().equals(scope.getTemplateId()); + } else if (scope.getDossierId() != null && scope.getDossierId().length() > 0) { + return image.getDossierId() != null && image.getDossierId().equals(scope.getDossierId()); + } else if (scope.getFileId() != null && scope.getFileId().length() > 0) { + return image.getFileId() != null && image.getFileId().equals(scope.getFileId()); + } + return true; + } + + + private double calculateManhattanDistance(double[] vector1, double[] vector2) { + + double distance = 0.0; + for (int i = 0; i < vector1.length; i++) { + distance += Math.abs(vector1[i] - vector2[i]); + } + return distance; + } + + + public void saveImages(List imageDocuments) { + + imageDocumentRepository.saveAll(imageDocuments); + } + + + @Getter + private static class ImageDistancePair { + + private final ImageDocument imageDocument; + private final double distance; + + + public ImageDistancePair(ImageDocument imageDocument, double distance) { + + this.imageDocument = imageDocument; + this.distance = distance; + } + + } + +} -- 2.47.2 From 0cdd10e98560e2d564fcca7f889d2f84bda6683c Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:19:10 +0200 Subject: [PATCH 06/15] RED-3813: image recategorization fixed merge with main --- .../service/queue/ImageMessageReceiver.java | 1 - .../LayoutParsingFinishedMessageReceiver.java | 31 ++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 64ba4d923..616d2bf50 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -47,7 +47,6 @@ public class ImageMessageReceiver { fileStatusService.setStatusAnalyse(dossierId, fileId, false); log.info("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_RESPONSE_QUEUE, dossierId, fileId); - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 9a45adb74..a4f9e7568 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -1,14 +1,25 @@ package com.iqser.red.service.persistence.management.v1.processor.service.queue; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.exc.ValueInstantiationException; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService; +import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService; +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; @@ -17,13 +28,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.core.Message; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; - @Slf4j @Service @RequiredArgsConstructor @@ -36,6 +40,7 @@ public class LayoutParsingFinishedMessageReceiver { private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; private final SaasMigrationService saasMigrationService; private final ImageSimilarityService imageSimiliarityService; + private final WebsocketService websocketService; @SneakyThrows @@ -48,23 +53,21 @@ public class LayoutParsingFinishedMessageReceiver { return; } - String templateId = ""; - String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); - - - fileStatusService.setStatusAnalyse(dossierId, fileId, layoutParsingRequestIdentifierService.parsePriority(response.identifier())); + var templateId = ""; var dossierId = layoutParsingRequestIdentifierService.parseDossierId(response.identifier()); var fileId = layoutParsingRequestIdentifierService.parseFileId(response.identifier()); + var storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); fileStatusService.setStatusAnalyse(layoutParsingRequestIdentifierService.parseDossierId(response.identifier()), layoutParsingRequestIdentifierService.parseFileId(response.identifier()), layoutParsingRequestIdentifierService.parsePriority(response.identifier())); fileStatusService.updateLayoutProcessedTime(layoutParsingRequestIdentifierService.parseFileId(response.identifier())); - imageSimiliarityService.saveImages(templateId, dossierId, fileId, storageId); websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.LAYOUT_UPDATE, fileStatusService.getStatus(fileId).getNumberOfAnalyses() + 1); + imageSimiliarityService.saveImages(templateId, dossierId, fileId, storageId); log.info("Received message {} in {}", response, LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE); + } -- 2.47.2 From 18a8f5215fd089e2344c46e98ab562cba3a79504 Mon Sep 17 00:00:00 2001 From: yhampe Date: Mon, 13 May 2024 15:06:25 +0200 Subject: [PATCH 07/15] RED-3813: image recategorization added first simple clustering service --- .../service/image/ImageClusteringService.java | 61 +++++++++++++++++++ .../service/queue/ImageMessageReceiver.java | 1 + 2 files changed, 62 insertions(+) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java new file mode 100644 index 000000000..62bd9ed33 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.image; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.math3.ml.clustering.Cluster; +import org.apache.commons.math3.ml.clustering.DBSCANClusterer; +import org.apache.commons.math3.ml.distance.DistanceMeasure; +import org.apache.commons.math3.ml.distance.ManhattanDistance; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageClusteringService { + + private final ObjectMapper objectMapper; + private final double eps = 26; + private final int minPts = 3; + + private final int fixedLength = 25; + private final DistanceMeasure distanceMeasure = new ManhattanDistance(); + + @Autowired + FileManagementStorageService fileManagementStorageService; + + + public void clusterImages(String storageId) throws Exception { + + DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); + + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); + List imageMetadataList = imageServiceResponse.getData(); + List imageList = new ArrayList<>(); + for (ImageMetadata metadata : imageMetadataList) { + Image image = new Image(metadata.getRepresentation(), fixedLength); + imageList.add(image); + } + if (imageList.isEmpty()) { + throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); + } + List> clusters = dbscanClusterer.cluster(imageList); + for (Cluster cluster : clusters) { + List clusterImages = cluster.getPoints(); + log.info("cluster: {}", clusterImages.get(0)); + } + } + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index 616d2bf50..d9d04d628 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -58,6 +58,7 @@ public class ImageMessageReceiver { String dossierId = (String) imageResponse.get("dossierId"); String fileId = (String) imageResponse.get("fileId"); addFileIdToTrace(fileId); + log.warn("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_DLQ, dossierId, fileId); fileStatusProcessingUpdateService.analysisFailed(dossierId, fileId, -- 2.47.2 From 7e65396ffe245fbb725348c6b2a35a7a29599725 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:03:13 +0200 Subject: [PATCH 08/15] RED-3813: image recategorization added saving images after layout parsing finishes added endpoint for querying similiar images --- .../service/image/ImageClusteringService.java | 61 ------------------- .../service/queue/ImageMessageReceiver.java | 1 + .../LayoutParsingFinishedMessageReceiver.java | 9 ++- 3 files changed, 9 insertions(+), 62 deletions(-) delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java deleted file mode 100644 index 62bd9ed33..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageClusteringService.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.math3.ml.clustering.Cluster; -import org.apache.commons.math3.ml.clustering.DBSCANClusterer; -import org.apache.commons.math3.ml.distance.DistanceMeasure; -import org.apache.commons.math3.ml.distance.ManhattanDistance; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.knecon.fforesight.tenantcommons.TenantContext; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class ImageClusteringService { - - private final ObjectMapper objectMapper; - private final double eps = 26; - private final int minPts = 3; - - private final int fixedLength = 25; - private final DistanceMeasure distanceMeasure = new ManhattanDistance(); - - @Autowired - FileManagementStorageService fileManagementStorageService; - - - public void clusterImages(String storageId) throws Exception { - - DBSCANClusterer dbscanClusterer = new DBSCANClusterer<>(eps, minPts, distanceMeasure); - - try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { - ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class); - List imageMetadataList = imageServiceResponse.getData(); - List imageList = new ArrayList<>(); - for (ImageMetadata metadata : imageMetadataList) { - Image image = new Image(metadata.getRepresentation(), fixedLength); - imageList.add(image); - } - if (imageList.isEmpty()) { - throw new IllegalArgumentException("The image list is empty. Unable to perform clustering."); - } - List> clusters = dbscanClusterer.cluster(imageList); - for (Cluster cluster : clusters) { - List clusterImages = cluster.getPoints(); - log.info("cluster: {}", clusterImages.get(0)); - } - } - - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java index d9d04d628..a1807c6b5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/ImageMessageReceiver.java @@ -47,6 +47,7 @@ public class ImageMessageReceiver { fileStatusService.setStatusAnalyse(dossierId, fileId, false); log.info("Received message from {} for dossierId {} and fileId {}", MessagingConfiguration.IMAGE_SERVICE_RESPONSE_QUEUE, dossierId, fileId); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index a4f9e7568..1fe318e96 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -13,6 +13,7 @@ import com.iqser.red.service.persistence.management.v1.processor.migration.SaasM import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService; import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; @@ -28,6 +29,13 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + @Slf4j @Service @RequiredArgsConstructor @@ -67,7 +75,6 @@ public class LayoutParsingFinishedMessageReceiver { imageSimiliarityService.saveImages(templateId, dossierId, fileId, storageId); log.info("Received message {} in {}", response, LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE); - } -- 2.47.2 From c2c6438187f6038b802c6ee9f329c54c09b2ab53 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:23:36 +0200 Subject: [PATCH 09/15] RED-3813: image recategorization merged with main --- .../queue/LayoutParsingFinishedMessageReceiver.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 1fe318e96..2c1c8d146 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -13,7 +13,6 @@ import com.iqser.red.service.persistence.management.v1.processor.migration.SaasM import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService; import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; @@ -29,13 +28,6 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.core.Message; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; - @Slf4j @Service @RequiredArgsConstructor -- 2.47.2 From 80be82e361402c749088c7d05803dd6251381c05 Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:27:07 +0200 Subject: [PATCH 10/15] RED-3813: image recategorization removed not used changes --- .../service/image/Classification.java | 20 ---- .../service/image/FilterGeometry.java | 17 ---- .../v1/processor/service/image/Filters.java | 18 ---- .../v1/processor/service/image/Geometry.java | 17 ---- .../v1/processor/service/image/Image.java | 26 ----- .../processor/service/image/ImageFormat.java | 18 ---- .../service/image/ImageMetadata.java | 21 ---- .../service/image/ImageServiceResponse.java | 34 ------- .../service/image/ImageSimilarityService.java | 96 ------------------- .../v1/processor/service/image/ImageSize.java | 18 ---- .../v1/processor/service/image/Position.java | 20 ---- .../processor/service/image/Probability.java | 16 ---- .../service/ImageClusteringTest.java | 5 - 13 files changed, 326 deletions(-) delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java delete mode 100644 persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java deleted file mode 100644 index 42ac1de40..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Classification.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import java.util.HashMap; -import java.util.Map; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class Classification { - - private Map probabilities = new HashMap<>(); - private String label; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java deleted file mode 100644 index 7a8f248c8..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/FilterGeometry.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class FilterGeometry { - - private ImageSize imageSize; - private ImageFormat imageFormat; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java deleted file mode 100644 index 7c31d6b07..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Filters.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class Filters { - - private FilterGeometry geometry; - private Probability probability; - private boolean allPassed; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java deleted file mode 100644 index 78ac75668..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Geometry.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class Geometry { - - private float width; - private float height; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java deleted file mode 100644 index ca807331c..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Image.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import org.apache.commons.math3.ml.clustering.Clusterable; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@AllArgsConstructor -public class Image implements Clusterable { - private String hash; - private int fixedLength; - private String label; - - @Override - public double[] getPoint() { - double[] featureVector = new double[fixedLength]; - for (int i = 0; i < hash.length(); i++) { - featureVector[i] = (double) hash.charAt(i); - } - return featureVector; - } -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java deleted file mode 100644 index 89bd37ad2..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageFormat.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ImageFormat { - - private float quotient; - private boolean tooTall; - private boolean tooWide; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java deleted file mode 100644 index 5e7c59d50..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageMetadata.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ImageMetadata { - - private Classification classification; - private String representation; - private Position position; - private Geometry geometry; - private Filters filters; - private boolean alpha; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java deleted file mode 100644 index e2e548002..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageServiceResponse.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ImageServiceResponse { - - private String dossierId; - private String fileId; - - @JsonProperty(value = "imageMetadata") - @JsonAlias("data") - private List data = new ArrayList<>(); - - private List dataCV = new ArrayList<>(); - - - @JsonProperty(value = "imageMetadata") - @JsonAlias("data") - public void setData(List data) {this.data = data;} - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java deleted file mode 100644 index f9111a584..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSimilarityService.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; -import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; -import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; -import com.knecon.fforesight.tenantcommons.TenantContext; - -import lombok.extern.slf4j.Slf4j; - -@Service -@Slf4j -public class ImageSimilarityService { - - @Autowired - ImageMongoService imageMongoService; - @Autowired - FileManagementStorageService fileManagementStorageService; - @Autowired - ObjectMapper objectMapper; - - - //load all recat requests and find similiar images - public void saveImages(String templateId, String dossierId, String fileId, String storageId) throws IOException { - //load structure files of all files in template - List imageDocuments = new ArrayList<>(); - try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { - //load images from structures - DocumentStructure documentStructure = objectMapper.readValue(inputStream, DocumentStructure.class); - documentStructure.streamAllEntries() - .filter(entry -> entry.getType().equals(NodeType.IMAGE)) - .forEach(i -> { - Map properties = i.getProperties(); - ImageDocument imageDocument = new ImageDocument(); - imageDocument.setImageId(properties.get("id")); - imageDocument.setFeatureVector(parseRepresentationVector(properties.get("representationHash"))); - imageDocument.setTemplateId(templateId); - imageDocument.setDossierId(dossierId); - imageDocument.setFileId(fileId); - imageDocuments.add(imageDocument); - - }); - } - if (imageDocuments.isEmpty()) { - return; - } - imageMongoService.saveImages(imageDocuments); - } - - - public List findSimilarImages(String centroId, double distance, Scope scope) throws Exception { - - ImageDocument centroImage = this.imageMongoService.findById(centroId); - log.info("image received with id {}: {}",centroId, centroImage); - List similarImages = this.imageMongoService.findSimilarImages(centroImage, distance, scope); - log.info("received similar images: {}",similarImages); - return similarImages.stream() - .collect(Collectors.toList()); - } - - - public static double[] parseRepresentationVector(String representationHash) { - - double[] doubleArray = new double[representationHash.length()]; - - for (int i = 0; i < representationHash.length(); i++) { - char c = representationHash.charAt(i); - if (Character.isDigit(c)) { - // Convert numeric characters directly to their numeric values - doubleArray[i] = Character.getNumericValue(c); - } else if (Character.isLetter(c)) { - // Convert alphabetic characters to their position in the alphabet - // 'A' or 'a' -> 10, 'B' or 'b' -> 11, ..., 'F' or 'f' -> 15 - doubleArray[i] = 10 + Character.toUpperCase(c) - 'A'; - } else { - throw new IllegalArgumentException("Invalid character in input string: " + c); - } - } - - return doubleArray; - } - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java deleted file mode 100644 index 40343e0e3..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/ImageSize.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ImageSize { - - private float quotient; - private boolean tooLarge; - private boolean tooSmall; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java deleted file mode 100644 index 31b347f0e..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Position.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class Position { - - private float x1; - private float x2; - private float y1; - private float y2; - private int pageNumber; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java deleted file mode 100644 index daec73d1d..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/image/Probability.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service.image; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class Probability { - - private boolean unconfident; - -} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java b/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java deleted file mode 100644 index dad72ea06..000000000 --- a/persistence-service-v1/persistence-service-processor-v1/src/test/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageClusteringTest.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.iqser.red.service.persistence.management.v1.processor.service; - -public class ImageClusteringTest { - -} -- 2.47.2 From 767f77f2618b39df84428bdb125c5b50a540c22d Mon Sep 17 00:00:00 2001 From: yhampe Date: Wed, 26 Jun 2024 15:28:47 +0200 Subject: [PATCH 11/15] RED-3813: image recategorization removed not used changes --- .../persistence-service-processor-v1/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 634002c14..05f998c98 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -63,7 +63,6 @@ dependencies { api("com.opencsv:opencsv:5.9") implementation("org.mapstruct:mapstruct:1.5.5.Final") - implementation("org.apache.commons:commons-math3:3.6.1") annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2") -- 2.47.2 From 7a24702c478b3aa26369255dd4e71369dcafe175 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 27 Jun 2024 12:16:43 +0200 Subject: [PATCH 12/15] RED-3813: image recategorization readded deleted service --- .../service/ImageSimilarityService.java | 96 +++++++++++++++++++ .../LayoutParsingFinishedMessageReceiver.java | 6 +- 2 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java new file mode 100644 index 000000000..e0e538994 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java @@ -0,0 +1,96 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class ImageSimilarityService { + + @Autowired + ImageMongoService imageMongoService; + @Autowired + FileManagementStorageService fileManagementStorageService; + @Autowired + ObjectMapper objectMapper; + + + //load all recat requests and find similiar images + public void saveImages(String templateId, String dossierId, String fileId, String storageId) throws IOException { + //load structure files of all files in template + List imageDocuments = new ArrayList<>(); + try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { + //load images from structures + DocumentStructure documentStructure = objectMapper.readValue(inputStream, DocumentStructure.class); + documentStructure.streamAllEntries() + .filter(entry -> entry.getType().equals(NodeType.IMAGE)) + .forEach(i -> { + Map properties = i.getProperties(); + ImageDocument imageDocument = new ImageDocument(); + imageDocument.setImageId(properties.get("id")); + imageDocument.setFeatureVector(parseRepresentationVector(properties.get("representationHash"))); + imageDocument.setTemplateId(templateId); + imageDocument.setDossierId(dossierId); + imageDocument.setFileId(fileId); + imageDocuments.add(imageDocument); + + }); + } + if (imageDocuments.isEmpty()) { + return; + } + imageMongoService.saveImages(imageDocuments); + } + + + public List findSimilarImages(String centroId, double distance, Scope scope) throws Exception { + + ImageDocument centroImage = this.imageMongoService.findById(centroId); + log.info("image received with id {}: {}",centroId, centroImage); + List similarImages = this.imageMongoService.findSimilarImages(centroImage, distance, scope); + log.info("received similar images: {}",similarImages); + return similarImages.stream() + .collect(Collectors.toList()); + } + + + public static double[] parseRepresentationVector(String representationHash) { + + double[] doubleArray = new double[representationHash.length()]; + + for (int i = 0; i < representationHash.length(); i++) { + char c = representationHash.charAt(i); + if (Character.isDigit(c)) { + // Convert numeric characters directly to their numeric values + doubleArray[i] = Character.getNumericValue(c); + } else if (Character.isLetter(c)) { + // Convert alphabetic characters to their position in the alphabet + // 'A' or 'a' -> 10, 'B' or 'b' -> 11, ..., 'F' or 'f' -> 15 + doubleArray[i] = 10 + Character.toUpperCase(c) - 'A'; + } else { + throw new IllegalArgumentException("Invalid character in input string: " + c); + } + } + + return doubleArray; + } + +} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java index 2c1c8d146..7035abacc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/LayoutParsingFinishedMessageReceiver.java @@ -13,8 +13,8 @@ import com.iqser.red.service.persistence.management.v1.processor.migration.SaasM import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.WebsocketService; -import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.LayoutParsingRequestIdentifierService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.SaasMigrationStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; @@ -39,7 +39,7 @@ public class LayoutParsingFinishedMessageReceiver { private final LayoutParsingRequestIdentifierService layoutParsingRequestIdentifierService; private final SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService; private final SaasMigrationService saasMigrationService; - private final ImageSimilarityService imageSimiliarityService; + private final ImageSimilarityService imageSimilarityService; private final WebsocketService websocketService; @@ -64,7 +64,7 @@ public class LayoutParsingFinishedMessageReceiver { fileStatusService.updateLayoutProcessedTime(layoutParsingRequestIdentifierService.parseFileId(response.identifier())); websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.LAYOUT_UPDATE, fileStatusService.getStatus(fileId).getNumberOfAnalyses() + 1); - imageSimiliarityService.saveImages(templateId, dossierId, fileId, storageId); + imageSimilarityService.saveImages(templateId, dossierId, fileId, storageId); log.info("Received message {} in {}", response, LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE); } -- 2.47.2 From 44bce39a2ab814df8fbce959b0eb19e43d4ced24 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 27 Jun 2024 12:19:45 +0200 Subject: [PATCH 13/15] RED-3813: image recategorization refactor checkstyle --- .../ImageSimilaritySearchController.java | 16 ++++++++-------- .../service/ImageSimilarityService.java | 9 +++------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java index 23839b430..411f44eb8 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java @@ -11,10 +11,10 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import com.iqser.red.service.persistence.management.v1.processor.service.image.ImageSimilarityService; +import com.iqser.red.service.persistence.management.v1.processor.service.ImageSimilarityService; import com.iqser.red.service.persistence.service.v1.api.external.resource.ImageSimilaritySearchResource; -import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.image.ImageSimilaritySearchResponse; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; import lombok.RequiredArgsConstructor; @@ -26,17 +26,17 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class ImageSimilaritySearchController implements ImageSimilaritySearchResource { - private final ImageSimilarityService imageSimiliarityService; + private final ImageSimilarityService imageSimilarityService; @SneakyThrows + @PreAuthorize("hasAuthority('" + GET_SIMILIAR_IMAGES + "')") public ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest) { - - log.info("received similiar image search request {}",imageSimilaritySearchRequest); - List similarImages = this.imageSimiliarityService.findSimilarImages(imageSimilaritySearchRequest.getCentroId(), - imageSimilaritySearchRequest.getDistance(), - imageSimilaritySearchRequest.getScope()); + log.info("received similiar image search request {}", imageSimilaritySearchRequest); + List similarImages = this.imageSimilarityService.findSimilarImages(imageSimilaritySearchRequest.getCentroId(), + imageSimilaritySearchRequest.getDistance(), + imageSimilaritySearchRequest.getScope()); List similarImagesIds = new ArrayList<>(); List similarImagesThumbnails = new ArrayList<>(); similarImages.stream() diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java index e0e538994..983c3951d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ImageSimilarityService.java @@ -11,7 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.Scope; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.ImageDocument; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ImageMongoService; @@ -33,12 +32,10 @@ public class ImageSimilarityService { ObjectMapper objectMapper; - //load all recat requests and find similiar images public void saveImages(String templateId, String dossierId, String fileId, String storageId) throws IOException { - //load structure files of all files in template + List imageDocuments = new ArrayList<>(); try (InputStream inputStream = fileManagementStorageService.getObject(TenantContext.getTenantId(), storageId)) { - //load images from structures DocumentStructure documentStructure = objectMapper.readValue(inputStream, DocumentStructure.class); documentStructure.streamAllEntries() .filter(entry -> entry.getType().equals(NodeType.IMAGE)) @@ -64,9 +61,9 @@ public class ImageSimilarityService { public List findSimilarImages(String centroId, double distance, Scope scope) throws Exception { ImageDocument centroImage = this.imageMongoService.findById(centroId); - log.info("image received with id {}: {}",centroId, centroImage); + log.info("image received with id {}: {}", centroId, centroImage); List similarImages = this.imageMongoService.findSimilarImages(centroImage, distance, scope); - log.info("received similar images: {}",similarImages); + log.info("received similar images: {}", similarImages); return similarImages.stream() .collect(Collectors.toList()); } -- 2.47.2 From c3aaba45ccda785e828566125bf98c371f725def Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 27 Jun 2024 12:20:55 +0200 Subject: [PATCH 14/15] RED-3813: image recategorization refactor checkstyle --- .../api/impl/controller/ImageSimilaritySearchController.java | 4 ++-- .../management/v1/processor/roles/ActionRoles.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java index 411f44eb8..62998a489 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ImageSimilaritySearchController.java @@ -1,6 +1,6 @@ package com.iqser.red.persistence.service.v1.external.api.impl.controller; -import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_SIMILIAR_IMAGES; +import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.GET_SIMILAR_IMAGES; import java.util.ArrayList; import java.util.List; @@ -30,7 +30,7 @@ public class ImageSimilaritySearchController implements ImageSimilaritySearchRes @SneakyThrows - @PreAuthorize("hasAuthority('" + GET_SIMILIAR_IMAGES + "')") + @PreAuthorize("hasAuthority('" + GET_SIMILAR_IMAGES + "')") public ResponseEntity getSimilarImages(@RequestBody ImageSimilaritySearchRequest imageSimilaritySearchRequest) { log.info("received similiar image search request {}", imageSimilaritySearchRequest); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java index 8113193a0..cb81b38d3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/roles/ActionRoles.java @@ -61,7 +61,7 @@ public final class ActionRoles { // IMAGE SIMILIARITY SEARCH - public static final String GET_SIMILIAR_IMAGES ="red-get-similiar-images"; + public static final String GET_SIMILAR_IMAGES ="red-get-similiar-images"; // Preferences public static final String MANAGE_USER_PREFERENCES = "red-manage-user-preferences"; -- 2.47.2 From 361b11ca2bcf74dde16b730f5cc6f1b0cc492e48 Mon Sep 17 00:00:00 2001 From: yhampe Date: Thu, 27 Jun 2024 12:44:12 +0200 Subject: [PATCH 15/15] RED-3813: image recategorization checkstyle --- .../service/v1/api/shared/mongo/service/ImageMongoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java index 29517993c..1abbc9f98 100644 --- a/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java +++ b/persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ImageMongoService.java @@ -92,7 +92,7 @@ public class ImageMongoService { private final double distance; - public ImageDistancePair(ImageDocument imageDocument, double distance) { + ImageDistancePair(ImageDocument imageDocument, double distance) { this.imageDocument = imageDocument; this.distance = distance; -- 2.47.2