Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-8043' into 'master'

Browse Source 
RED-8043 - Refactor EntityLog merge mechanism

Closes RED-8043

See merge request redactmanager/persistence-service!269
This commit is contained in:
Andrei Isvoran 2023-12-14 15:06:32 +01:00
commit b0f40a080c
15 changed files with 254 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts
View File

@ -6,6 +6,11 @@ plugins {
jacoco
}
val redactionServiceVersion by rootProject.extra { "4.199.0" }
val pdftronRedactionServiceVersion by rootProject.extra { "4.38.0" }
val redactionReportServiceVersion by rootProject.extra { "4.36.0" }
val searchServiceVersion by rootProject.extra { "2.71.0" }
repositories {
mavenLocal()
mavenCentral()

9
persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts
View File

@ -4,19 +4,19 @@ plugins {
dependencies {
api(project(":persistence-service-internal-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") {
api("com.iqser.red.service:pdftron-redaction-service-api-v1:${rootProject.extra.get("pdftronRedactionServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-service-api-v1:4.196.0") {
api("com.iqser.red.service:redaction-service-api-v1:${rootProject.extra.get("redactionServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") {
api("com.iqser.red.service:redaction-report-service-api-v1:${rootProject.extra.get("redactionReportServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:search-service-api-v1:2.71.0") {
api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
@ -34,3 +34,4 @@ dependencies {
}
description = "persistence-service-external-api-v1"

8
persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts
View File

@ -6,19 +6,19 @@ plugins {
dependencies {
api(project(":persistence-service-external-api-v1"))
api(project(":persistence-service-internal-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") {
api("com.iqser.red.service:pdftron-redaction-service-api-v1:${rootProject.extra.get("pdftronRedactionServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-service-api-v1:4.196.0") {
api("com.iqser.red.service:redaction-service-api-v1:${rootProject.extra.get("redactionServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") {
api("com.iqser.red.service:redaction-report-service-api-v1:${rootProject.extra.get("redactionReportServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:search-service-api-v1:2.71.0") {
api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}

8
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -9,15 +9,15 @@ dependencies {
api(project(":persistence-service-external-api-v1"))
api(project(":persistence-service-internal-api-v1"))
api(project(":persistence-service-shared-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") {
api("com.iqser.red.service:pdftron-redaction-service-api-v1:${rootProject.extra.get("pdftronRedactionServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-service-api-v1:4.196.0") {
api("com.iqser.red.service:redaction-service-api-v1:${rootProject.extra.get("redactionServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") {
api("com.iqser.red.service:redaction-report-service-api-v1:${rootProject.extra.get("redactionReportServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
@ -25,7 +25,7 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:search-service-api-v1:2.71.0") {
api("com.iqser.red.service:search-service-api-v1:${rootProject.extra.get("searchServiceVersion")}") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/client/redactionservice/UnprocessedManualEntityClient.java
View File

@ -1,10 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.client.redactionservice;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
@FeignClient(name = "UnprocessedManualEntityClient", url = "${redaction-service.url}")
public interface UnprocessedManualEntityClient extends UnprocessedManualEntityResource {
}

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/configuration/MessagingConfiguration.java
View File

@ -182,20 +182,19 @@ public class MessagingConfiguration {
}
@Bean
public Queue redactionAnalysisResponseQueue() {
public Queue redactionPriorityQueue() {
return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
.maxPriority(2)
.build();
}
@Bean
public Queue redactionPriorityQueue() {
public Queue redactionAnalysisResponseQueue() {
return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
.withArgument("x-dead-letter-exchange", "")
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
.maxPriority(2)

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/UncompressedFilesMigrationService.java
View File

@ -88,10 +88,11 @@ public class UncompressedFilesMigrationService {
client.getS3StorageClient().getObject(getObjectRequest, tempFile.toPath());
var fis = new FileInputStream(tempFile);
storageService.storeObject(tenant, key, fis);
try(var fis = new FileInputStream(tempFile)) {
storageService.storeObject(tenant, key, fis);
IOUtils.closeQuietly(fis);
IOUtils.closeQuietly(fis);
}
try {
tempFile.delete();
} catch (Exception e) {

129
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -9,14 +9,21 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -35,8 +42,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -47,23 +54,30 @@ import lombok.extern.slf4j.Slf4j;
public class EntityLogMergeService {
private final DictionaryPersistenceService dictionaryPersistenceService;
private final RabbitTemplate rabbitTemplate;
private final FileStatusService fileStatusService;
private final FileStatusPersistenceService fileStatusPersistenceService;
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier) {
public EntityLog mergeEntityLog(ManualRedactions manualRedactions, EntityLog entityLog, DossierEntity dossier, FileModel fileModel) {
log.info("Merging EntityLog");
List<BaseAnnotation> allManualChanges = allManualChanges(manualRedactions);
ManualRedactions manualRedactionsWhichRequireExtraInformation = ManualRedactions.builder()
.entriesToAdd(new HashSet<>())
.resizeRedactions(new HashSet<>())
.build();
// Sort manual changes by date so we process them in order of when they were requested
// Sort manual changes by date, so we process them in order of when they were requested
allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
allManualChanges.forEach(manualChange -> {
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
if (manualChange instanceof ManualRedactionEntry) {
mergeManualRedactionEntries((ManualRedactionEntry) manualChange, unprocessedManualEntities, entityLog, dossier);
mergeManualRedactionEntries((ManualRedactionEntry) manualChange, entityLog, dossier, manualRedactionsWhichRequireExtraInformation);
} else if (manualChange instanceof IdRemoval) {
mergeIdsToRemove((IdRemoval) manualChange, entityLog);
} else if (manualChange instanceof ManualResizeRedaction) {
mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog);
mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog, manualRedactionsWhichRequireExtraInformation);
} else if (manualChange instanceof ManualLegalBasisChange) {
mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog);
} else if (manualChange instanceof ManualRecategorization) {
@ -73,19 +87,21 @@ public class EntityLogMergeService {
}
});
log.info("EntityLog merged successfully!");
if (!manualRedactionsWhichRequireExtraInformation.getEntriesToAdd().isEmpty() || !manualRedactionsWhichRequireExtraInformation.getResizeRedactions().isEmpty()) {
sendToAnalyseQueue(fileModel.getId(), dossier, fileModel, manualRedactionsWhichRequireExtraInformation);
}
log.info("EntityLog merged successfully.");
return entityLog;
}
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, List<UnprocessedManualEntity> unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier) {
private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, EntityLog entityLog, DossierEntity dossier, ManualRedactions manualRedactions) {
if (manualRedactionEntry.getPositions() == null || manualRedactionEntry.getPositions().isEmpty()) {
return;
}
UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream()
.filter(manualEntity -> manualEntity.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
.findFirst().orElseThrow(() -> new NotFoundException("Entry with annotationId " + manualRedactionEntry.getAnnotationId() + " not found"));
List<Change> changes = new ArrayList<>();
changes.add(Change.builder()
@ -106,31 +122,27 @@ public class EntityLogMergeService {
return;
}
var entityLogEntryBuilder = EntityLogEntry.builder()
entityLog.getEntityLogEntry().add(EntityLogEntry.builder()
.id(manualRedactionEntry.getAnnotationId())
.type(manualRedactionEntry.getType())
.value(manualRedactionEntry.getValue())
.legalBasis(manualRedactionEntry.getLegalBasis() == null || manualRedactionEntry.getLegalBasis().isEmpty() ? unprocessedManualEntity.getLegalBasis() : manualRedactionEntry.getLegalBasis())
.legalBasis(manualRedactionEntry.getLegalBasis())
.reason(manualRedactionEntry.getReason())
.entryType(isHint ? EntryType.HINT : EntryType.ENTITY)
.state(isHint ? EntryState.SKIPPED : EntryState.APPLIED)
.imported(false)
.matchedRule("")
.section(manualRedactionEntry.getSection())
.positions(unprocessedManualEntity.getPositions())
.textAfter(unprocessedManualEntity.getTextAfter())
.textBefore(unprocessedManualEntity.getTextBefore())
.startOffset(unprocessedManualEntity.getStartOffset())
.endOffset(unprocessedManualEntity.getEndOffset())
.containingNodeId(unprocessedManualEntity.getContainingNodeId())
.closestHeadline(unprocessedManualEntity.getClosestHeadline())
.textBefore(manualRedactionEntry.getTextBefore())
.textAfter(manualRedactionEntry.getTextAfter())
.positions(convertPositions(manualRedactionEntry.getPositions()))
.imageHasTransparency(false)
.dictionaryEntry(manualRedactionEntry.isAddToDictionary())
.dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.excluded(false)
.changes(changes)
.manualChanges(List.of(ManualChange.builder()
.manualRedactionType(ManualRedactionType.ADD_LOCALLY)
.manualRedactionType(calculateManualRedactionType(manualRedactionEntry))
.requestedDate(manualRedactionEntry.getRequestDate())
.processedDate(null)
.userId(manualRedactionEntry.getUser())
@ -138,14 +150,14 @@ public class EntityLogMergeService {
.build()))
.engines(new HashSet<>())
.reference(new HashSet<>())
.importedRedactionIntersections(new HashSet<>());
entityLog.getEntityLogEntry().add(entityLogEntryBuilder
.importedRedactionIntersections(new HashSet<>())
.build());
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
}
private static boolean isFalsePositive(ManualRedactionEntry manualRedactionEntry) {
private boolean isFalsePositive(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getDictionaryEntryType() != null && manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE);
}
@ -183,24 +195,23 @@ public class EntityLogMergeService {
}
}
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog) {
private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog, ManualRedactions manualRedactions) {
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
if (entity.isPresent()) {
var newPosition = manualResizeRedaction.getPositions().get(0);
entity.get().setPositions(List.of(new Position(
newPosition.getTopLeftX(),
newPosition.getTopLeftY(),
newPosition.getWidth(),
newPosition.getHeight(),
newPosition.getPage())));
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entity.get().getManualChanges().add(ManualChange.builder()
EntityLogEntry entityLogEntry = entity.get();
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RESIZE)
.requestedDate(manualResizeRedaction.getRequestDate())
.processedDate(null)
.propertyChanges(Map.of("value", manualResizeRedaction.getValue()))
.userId(manualResizeRedaction.getUser()).build());
manualRedactions.getResizeRedactions().add(manualResizeRedaction);
}
}
@ -227,13 +238,14 @@ public class EntityLogMergeService {
private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) {
boolean isHint = isHint(recategorization.getType(), dossier);
var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
if (entity.isPresent()) {
entity.get().setType(recategorization.getType());
entity.get().setEntryType(getEntryType(isHint, recategorization.getType()));
entity.get().setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entity.get().getManualChanges().add(ManualChange.builder()
var optionalEntityLogEntry = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny();
if (optionalEntityLogEntry.isPresent()) {
var entity = optionalEntityLogEntry.get();
entity.setType(recategorization.getType());
entity.setEntryType(getEntryType(isHint, recategorization.getType()));
entity.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entity.getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber());
entity.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.RECATEGORIZE)
.requestedDate(recategorization.getRequestDate())
.processedDate(recategorization.getProcessedDate())
@ -244,7 +256,7 @@ public class EntityLogMergeService {
}
private static EntryType getEntryType(boolean isHint, String type) {
private EntryType getEntryType(boolean isHint, String type) {
if (type.equals("image") || type.equals("logo") || type.equals("signature") || type.equals("formula")) {
return isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE;
@ -253,6 +265,14 @@ public class EntityLogMergeService {
}
}
private ManualRedactionType calculateManualRedactionType(ManualRedactionEntry manualRedactionEntry) {
if (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) {
return ManualRedactionType.ADD_TO_DICTIONARY;
}
return ManualRedactionType.ADD_LOCALLY;
}
private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) {
@ -311,6 +331,11 @@ public class EntityLogMergeService {
&& position1.getHeight() == position2.h();
}
private List<Position> convertPositions(List<Rectangle> rectangles) {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
}
private List<BaseAnnotation> allManualChanges(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getEntriesToAdd(),
@ -321,4 +346,24 @@ public class EntityLogMergeService {
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
}
private void sendToAnalyseQueue(String fileId, DossierEntity dossier, FileModel fileModel, ManualRedactions manualRedactions) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
var analyseRequest = AnalyzeRequest.builder()
.messageType(MessageType.SURROUNDING_TEXT_ANALYSIS)
.dossierId(dossier.getId())
.analysisNumber(fileModel.getNumberOfAnalyses() + 1)
.sectionsToReanalyse(Sets.newHashSet())
.fileId(fileId)
.manualRedactions(manualRedactions)
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileModel.getLastProcessed())
.fileAttributes(fileStatusService.convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build();
log.info("Sending Surrounding Text Analysis for unprocessed manual redactions: {} for file: {} ", manualRedactions, fileId);
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, analyseRequest);
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java
View File

@ -10,14 +10,12 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -34,7 +32,6 @@ public class EntityLogService {
FileManagementStorageService fileManagementStorageService;
FileStatusService fileStatusService;
ManualRedactionProviderService manualRedactionProviderService;
UnprocessedManualEntityClient unprocessedManualEntityClient;
DossierService dossierService;
CommentService commentService;
EntityLogMergeService entityLogMergeService;
@ -65,8 +62,7 @@ public class EntityLogService {
if (includeUnprocessed) {
DossierEntity dossier = dossierService.getDossierById(dossierId);
ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true);
List<UnprocessedManualEntity> unprocessedManualEntities = getUnprocessedManualEntities(dossierId, fileId, dossier.getDossierTemplateId(), manualRedactions);
entityLogMergeService.mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier);
entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, dossier, fileStatus);
}
Map<String, Integer> commentCountPerAnnotationId = commentService.getCommentCounts(fileId);
@ -112,10 +108,4 @@ public class EntityLogService {
return entityLog;
}
private List<UnprocessedManualEntity> getUnprocessedManualEntities(String dossierId, String fileId, String dossierTemplateId, ManualRedactions manualRedactions) {
return unprocessedManualEntityClient.mergeUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions);
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusProcessingUpdateService.java
View File

@ -39,7 +39,9 @@ public class FileStatusProcessingUpdateService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
switch (analyzeResult.getMessageType()) {
case SURROUNDING_TEXT_ANALYSIS:
break;
case ANALYSE:
case REANALYSE:
default:

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -358,7 +358,7 @@ public class FileStatusService {
}
private List<FileAttribute> convertAttributes(List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
public List<FileAttribute> convertAttributes(List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
List<FileAttribute> fileAttributeList = new ArrayList<>();

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -4,6 +4,7 @@ import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.springframework.beans.BeanUtils;
@ -45,6 +46,10 @@ public class AddRedactionPersistenceService {
}
public Optional<ManualRedactionEntryEntity> findById(String annotationId, String fileId) {
return manualRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId));
}
private List<RectangleEntity> convert(List<Rectangle> positions) {
@ -111,6 +116,13 @@ public class AddRedactionPersistenceService {
}
@Transactional
public void update(ManualRedactionEntryEntity manualRedactionEntry) {
manualRedactionRepository.saveAndFlush(manualRedactionEntry);
}
@Transactional
public void updateStatus(String fileId,
String annotationId,

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.springframework.beans.BeanUtils;
@ -87,6 +88,11 @@ public class ResizeRedactionPersistenceService {
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
}
public Optional<ManualResizeRedactionEntity> findResizeRedactionById(String fileId, String annotationId) {
return resizeRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId));
}
public List<ManualResizeRedactionEntity> findResizeRedactions(String fileId, boolean includeDeletions) {

89
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/RedactionAnalysisResponseReceiver.java Normal file
View File

@ -0,0 +1,89 @@
package com.iqser.red.service.persistence.management.v1.processor.service.queue;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public class RedactionAnalysisResponseReceiver {
ObjectMapper objectMapper;
AddRedactionPersistenceService addRedactionPersistenceService;
ResizeRedactionPersistenceService resizeRedactionPersistenceService;
@SneakyThrows
@RabbitHandler
@RabbitListener(queues = MessagingConfiguration.REDACTION_ANALYSIS_RESPONSE_QUEUE)
public void receive(Message message) throws JsonProcessingException {
AnalyzeResponse analyzeResponse = objectMapper.readValue(message.getBody(), new TypeReference<>() {});
String fileId = analyzeResponse.getFileId();
log.info("Received Surrounding Text Analysis response for file: {}", fileId);
analyzeResponse.getUnprocessedManualEntities().forEach(unprocessedManualEntity -> {
Optional<ManualRedactionEntryEntity> optionalManualRedactionEntry = addRedactionPersistenceService.findById(unprocessedManualEntity.getAnnotationId(), fileId);
if (optionalManualRedactionEntry.isPresent()) {
ManualRedactionEntryEntity manualRedactionEntry = optionalManualRedactionEntry.get();
if (!Objects.equals(manualRedactionEntry.getTextAfter(), unprocessedManualEntity.getTextAfter()) ||
!Objects.equals(manualRedactionEntry.getTextBefore(), unprocessedManualEntity.getTextBefore()) ||
!Objects.equals(manualRedactionEntry.getSection(), unprocessedManualEntity.getSection())) {
manualRedactionEntry.setTextBefore(unprocessedManualEntity.getTextBefore());
manualRedactionEntry.setTextAfter(unprocessedManualEntity.getTextAfter());
manualRedactionEntry.setSection(unprocessedManualEntity.getSection());
manualRedactionEntry.setPositions(convertPositions(unprocessedManualEntity.getPositions()));
addRedactionPersistenceService.update(manualRedactionEntry);
}
}
Optional<ManualResizeRedactionEntity> optionalManualResizeRedactionEntity = resizeRedactionPersistenceService.findResizeRedactionById(fileId, unprocessedManualEntity.getAnnotationId());
if (optionalManualResizeRedactionEntity.isPresent()) {
ManualResizeRedactionEntity manualResizeRedaction = optionalManualResizeRedactionEntity.get();
if (!Objects.equals(manualResizeRedaction.getTextAfter(), unprocessedManualEntity.getTextAfter()) ||
!Objects.equals(manualResizeRedaction.getTextBefore(), unprocessedManualEntity.getTextBefore())) {
resizeRedactionPersistenceService.updateSurroundingText(manualResizeRedaction.getId(),
unprocessedManualEntity.getTextBefore(),
unprocessedManualEntity.getTextAfter());
}
}
});
}
private List<RectangleEntity> convertPositions(List<Position> positions) {
return positions.stream().map(position -> RectangleEntity.builder()
.page(position.getPageNumber())
.height(position.h())
.width(position.w())
.topLeftX(position.x())
.topLeftY(position.y())
.build())
.toList();
}
}

55
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java
View File

@ -3,14 +3,15 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.UUID;
@ -18,23 +19,28 @@ import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mockito;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.testcontainers.shaded.com.google.common.collect.Lists;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.CommentService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@ -51,7 +57,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
@ExtendWith(SpringExtension.class)
public class EntityLogMergeTest {
@ -71,21 +76,27 @@ public class EntityLogMergeTest {
@MockBean
private ManualRedactionProviderService manualRedactionProviderService;
@MockBean
private UnprocessedManualEntityClient unprocessedManualEntityClient;
@MockBean
private DictionaryPersistenceService dictionaryPersistenceService;
@MockBean
private DossierTemplatePersistenceService dossierTemplatePersistenceService;
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private FileStatusPersistenceService fileStatusPersistenceService;
@Captor
private ArgumentCaptor<AnalyzeRequest> captor;
private EntityLogMergeService entityLogMergeService;
@BeforeEach
public void setUp() {
entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService);
entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService, rabbitTemplate, fileStatusService, fileStatusPersistenceService);
}
@ -103,21 +114,10 @@ public class EntityLogMergeTest {
String forceRedactionId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId);
UnprocessedManualEntity unprocessedManualEntity = UnprocessedManualEntity.builder()
.textBefore("textBefore")
.textAfter("textAfter")
.containingNodeId(List.of(0, 0))
.endOffset(10)
.startOffset(1)
.annotationId(entryToAddId)
.closestHeadline("closestHeadline")
.color(new float[]{0,0,0})
.build();
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId);
when(manualRedactionProviderService.getManualRedactions(fileId, true)).thenReturn(manualRedactions);
when(unprocessedManualEntityClient.mergeUnprocessedManualEntities(any(), any(), any(), any())).thenReturn(List.of(unprocessedManualEntity));
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder()
.excluded(false)
.dossierStatusId(dossierTemplateId)
@ -128,10 +128,17 @@ public class EntityLogMergeTest {
.dossierTemplateId(dossierTemplateId)
.build());
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, List.of(unprocessedManualEntity), entityLog, DossierEntity.builder()
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder()
.dossierTemplateId(dossierTemplateId)
.build());
.build(), FileModel.builder().id(fileId).build());
Mockito.verify(rabbitTemplate).convertAndSend(eq(MessagingConfiguration.REDACTION_PRIORITY_QUEUE), captor.capture());
AnalyzeRequest analyzeRequest = captor.getValue();
assertEquals(analyzeRequest.getManualRedactions().getEntriesToAdd().size(), 1);
assertEquals(analyzeRequest.getManualRedactions().getResizeRedactions().size(), 1);
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());
@ -144,10 +151,6 @@ public class EntityLogMergeTest {
assertEquals(entityLogEntry.getState(), EntryState.APPLIED);
assertEquals(entityLogEntry.getValue(), "Test");
assertEquals(entityLogEntry.getReason(), "Reason");
assertEquals(entityLogEntry.getTextAfter(), "textAfter");
assertEquals(entityLogEntry.getTextBefore(), "textBefore");
assertEquals(entityLogEntry.getStartOffset(), 1);
assertEquals(entityLogEntry.getEndOffset(), 10);
assertEquals(entityLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.ADD_LOCALLY);
var optionalRemoveEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRemoveId)).findFirst();