From 1cad386db83a2d2057701989c0c179b770248f5e Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 12 Jan 2022 20:50:00 +0200 Subject: [PATCH] analysis version in redactionlog and changes --- .../redaction/v1/model/AnalyzeRequest.java | 1 + .../redaction/v1/model/AnalyzeResult.java | 1 + .../service/redaction/v1/model/Change.java | 1 + .../redaction/v1/model/RedactionLog.java | 6 ++ .../redaction/service/AnalyzeService.java | 60 ++++++++----------- .../service/RedactionChangeLogService.java | 46 ++++++-------- 6 files changed, 51 insertions(+), 64 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 51ad598f..40635d5d 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -26,6 +26,7 @@ public class AnalyzeRequest { private String dossierTemplateId; private ManualRedactions manualRedactions; private OffsetDateTime lastProcessed; + private int analysisNumber; @Builder.Default private Set excludedPages = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index ec2aea13..dba1fdb9 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -28,6 +28,7 @@ public class AnalyzeResult { private boolean wasReanalyzed; private int analysisVersion; + private int analysisNumber; private ManualRedactions manualRedactions; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java index ebc59a84..96691071 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java @@ -13,6 +13,7 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class Change { + private int analysisNumber; private ChangeType type; private OffsetDateTime dateTime; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 908b5459..88de3a3a 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -18,6 +18,11 @@ public class RedactionLog { */ private long analysisVersion; + /** + * Which analysis created this redactionLog. + */ + private int analysisNumber; + private List redactionLogEntry; private List legalBasis; @@ -26,4 +31,5 @@ public class RedactionLog { private long rulesVersion = -1; private long legalBasisVersion = -1; + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index c9b28fde..32d4bf17 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -1,42 +1,31 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.model.Rectangle; -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestBody; - +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.segmentation.ImageService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Service @@ -71,7 +60,7 @@ public class AnalyzeService { .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN)); Map> pdfImages = null; - if(redactionServiceSettings.isEnableImageClassification()) { + if (redactionServiceSettings.isEnableImageClassification()) { pdfImages = imageService.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, pdfImages); @@ -83,7 +72,7 @@ public class AnalyzeService { List sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc); sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); - Text text = new Text(pageCount,sectionTexts); + Text text = new Text(pageCount, sectionTexts); // enhance section grid with headline data sectionTexts.forEach(sectionText -> classifiedDoc @@ -106,7 +95,7 @@ public class AnalyzeService { var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){ + if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) { nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } @@ -125,10 +114,9 @@ public class AnalyzeService { .getDossierTemplateId()); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary - .getVersion() - .getDossierTemplateVersion(), dictionary.getVersion() - .getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), + redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(), + rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false); } @@ -158,7 +146,7 @@ public class AnalyzeService { } var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){ + if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) { nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } @@ -223,7 +211,8 @@ public class AnalyzeService { excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); - var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), + redactionLog, analyzeRequest.getAnalysisNumber()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange .getRedactionLog()); @@ -236,6 +225,7 @@ public class AnalyzeService { .numberOfPages(text.getNumberOfPages()) .hasUpdates(redactionLogChange.isHasChanges()) .analysisVersion(redactionServiceSettings.getAnalysisVersion()) + .analysisNumber(analyzeRequest.getAnalysisNumber()) .rulesVersion(redactionLog.getRulesVersion()) .dictionaryVersion(redactionLog.getDictionaryVersion()) .legalBasisVersion(redactionLog.getLegalBasisVersion()) @@ -252,12 +242,12 @@ public class AnalyzeService { } return Stream.concat(manualRedactions.getLegalBasisChanges() - .stream() - .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization() - .stream() - .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove() - .stream() - .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))) + .stream() + .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization() + .stream() + .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove() + .stream() + .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))) .collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index ff3315f2..9ebb5f95 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -1,25 +1,14 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.model.Change; -import com.iqser.red.service.redaction.v1.model.ChangeType; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogChanges; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Service @@ -29,7 +18,7 @@ public class RedactionChangeLogService { private final RedactionStorageService redactionStorageService; - public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) { + public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) { long start = System.currentTimeMillis(); @@ -37,7 +26,7 @@ public class RedactionChangeLogService { if (previousRedactionLog == null) { currentRedactionLog.getRedactionLogEntry().forEach(entry -> { - entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); }); return new RedactionLogChanges(currentRedactionLog, false); } @@ -50,17 +39,16 @@ public class RedactionChangeLogService { .equals(ChangeType.REMOVED)) .collect(Collectors.toList()); - Set added = new HashSet<>(currentRedactionLog.getRedactionLogEntry().stream() + Set added = currentRedactionLog.getRedactionLogEntry().stream() .filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges() .get(entry.getChanges().size() - 1) .getType() - .equals(ChangeType.REMOVED)) - .collect(Collectors.toList())); + .equals(ChangeType.REMOVED)).collect(Collectors.toSet()); - added.removeAll(notRemovedPreviousEntries); + notRemovedPreviousEntries.forEach(added::remove); Set removed = new HashSet<>(notRemovedPreviousEntries); - removed.removeAll(currentRedactionLog.getRedactionLogEntry()); + currentRedactionLog.getRedactionLogEntry().forEach(removed::remove); Map addedIds = new HashMap<>(); added.forEach(entry -> { @@ -78,16 +66,16 @@ public class RedactionChangeLogService { newRedactionLogEntries.forEach(entry -> { if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) { List changes = entry.getChanges(); - changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now())); + changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now())); var newEntry = addedIds.get(entry.getId()); newEntry.setChanges(changes); addedIds.put(entry.getId(), newEntry); toRemove.add(entry); } else if (removedIds.contains(entry.getId())) { - entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now())); + entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); } else if (addedIds.containsKey(entry.getId())) { List changes = entry.getChanges(); - changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); var newEntry = addedIds.get(entry.getId()); newEntry.setChanges(changes); addedIds.put(entry.getId(), newEntry); @@ -98,8 +86,8 @@ public class RedactionChangeLogService { newRedactionLogEntries.removeAll(toRemove); addedIds.forEach((k, v) -> { - if(v.getChanges().isEmpty()) { - v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + if (v.getChanges().isEmpty()) { + v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now())); } newRedactionLogEntries.add(v); });