Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #318: analysis version in redactionlog and changes

Browse Source 
Merge in RED/redaction-service from analysis-number-in-redaction-log-m to master

* commit '1cad386db83a2d2057701989c0c179b770248f5e':
  analysis version in redactionlog and changes
This commit is contained in:
Timo Bejan 2022-01-13 09:23:34 +01:00
commit f05977b7ae
6 changed files with 51 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -26,6 +26,7 @@ public class AnalyzeRequest {
private String dossierTemplateId; private String dossierTemplateId;
private ManualRedactions manualRedactions; private ManualRedactions manualRedactions;
private OffsetDateTime lastProcessed; private OffsetDateTime lastProcessed;
private int analysisNumber;
@Builder.Default @Builder.Default
private Set<Integer> excludedPages = new HashSet<>(); private Set<Integer> excludedPages = new HashSet<>();

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java
View File

@ -28,6 +28,7 @@ public class AnalyzeResult {
private boolean wasReanalyzed; private boolean wasReanalyzed;
private int analysisVersion; private int analysisVersion;
private int analysisNumber;
private ManualRedactions manualRedactions; private ManualRedactions manualRedactions;

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java
View File

@ -13,6 +13,7 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor @NoArgsConstructor
public class Change { public class Change {
private int analysisNumber;
private ChangeType type; private ChangeType type;
private OffsetDateTime dateTime; private OffsetDateTime dateTime;
} }

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java
View File

@ -18,6 +18,11 @@ public class RedactionLog {
*/ */
private long analysisVersion; private long analysisVersion;
/**
* Which analysis created this redactionLog.
*/
private int analysisNumber;
private List<RedactionLogEntry> redactionLogEntry; private List<RedactionLogEntry> redactionLogEntry;
private List<LegalBasis> legalBasis; private List<LegalBasis> legalBasis;
@ -26,4 +31,5 @@ public class RedactionLog {
private long rulesVersion = -1; private long rulesVersion = -1;
private long legalBasisVersion = -1; private long legalBasisVersion = -1;
} }

60
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -1,42 +1,31 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
import org.kie.api.runtime.KieContainer; import com.iqser.red.service.redaction.v1.model.*;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService; import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j @Slf4j
@Service @Service
@ -71,7 +60,7 @@ public class AnalyzeService {
.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN)); .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
Map<Integer, List<PdfImage>> pdfImages = null; Map<Integer, List<PdfImage>> pdfImages = null;
if(redactionServiceSettings.isEnableImageClassification()) { if (redactionServiceSettings.isEnableImageClassification()) {
pdfImages = imageService.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); pdfImages = imageService.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
} }
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, pdfImages); classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, pdfImages);
@ -83,7 +72,7 @@ public class AnalyzeService {
List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc); List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
Text text = new Text(pageCount,sectionTexts); Text text = new Text(pageCount, sectionTexts);
// enhance section grid with headline data // enhance section grid with headline data
sectionTexts.forEach(sectionText -> classifiedDoc sectionTexts.forEach(sectionText -> classifiedDoc
@ -106,7 +95,7 @@ public class AnalyzeService {
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){ if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
} }
@ -125,10 +114,9 @@ public class AnalyzeService {
.getDossierTemplateId()); .getDossierTemplateId());
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
.getVersion() redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
.getDossierTemplateVersion(), dictionary.getVersion() rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
.getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
} }
@ -158,7 +146,7 @@ public class AnalyzeService {
} }
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){ if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
} }
@ -223,7 +211,8 @@ public class AnalyzeService {
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
redactionLog, analyzeRequest.getAnalysisNumber());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
.getRedactionLog()); .getRedactionLog());
@ -236,6 +225,7 @@ public class AnalyzeService {
.numberOfPages(text.getNumberOfPages()) .numberOfPages(text.getNumberOfPages())
.hasUpdates(redactionLogChange.isHasChanges()) .hasUpdates(redactionLogChange.isHasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion()) .analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(redactionLog.getRulesVersion()) .rulesVersion(redactionLog.getRulesVersion())
.dictionaryVersion(redactionLog.getDictionaryVersion()) .dictionaryVersion(redactionLog.getDictionaryVersion())
.legalBasisVersion(redactionLog.getLegalBasisVersion()) .legalBasisVersion(redactionLog.getLegalBasisVersion())
@ -252,12 +242,12 @@ public class AnalyzeService {
} }
return Stream.concat(manualRedactions.getLegalBasisChanges() return Stream.concat(manualRedactions.getLegalBasisChanges()
.stream() .stream()
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization() .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
.stream() .stream()
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove() .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
.stream() .stream()
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))) .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} }

46
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java
View File

@ -1,25 +1,14 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.time.OffsetDateTime; import com.iqser.red.service.redaction.v1.model.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Change;
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogChanges;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j @Slf4j
@Service @Service
@ -29,7 +18,7 @@ public class RedactionChangeLogService {
private final RedactionStorageService redactionStorageService; private final RedactionStorageService redactionStorageService;
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) { public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
@ -37,7 +26,7 @@ public class RedactionChangeLogService {
if (previousRedactionLog == null) { if (previousRedactionLog == null) {
currentRedactionLog.getRedactionLogEntry().forEach(entry -> { currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
}); });
return new RedactionLogChanges(currentRedactionLog, false); return new RedactionLogChanges(currentRedactionLog, false);
} }
@ -50,17 +39,16 @@ public class RedactionChangeLogService {
.equals(ChangeType.REMOVED)) .equals(ChangeType.REMOVED))
.collect(Collectors.toList()); .collect(Collectors.toList());
Set<RedactionLogEntry> added = new HashSet<>(currentRedactionLog.getRedactionLogEntry().stream() Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
.filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges() .filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges()
.get(entry.getChanges().size() - 1) .get(entry.getChanges().size() - 1)
.getType() .getType()
.equals(ChangeType.REMOVED)) .equals(ChangeType.REMOVED)).collect(Collectors.toSet());
.collect(Collectors.toList()));
added.removeAll(notRemovedPreviousEntries); notRemovedPreviousEntries.forEach(added::remove);
Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries); Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
removed.removeAll(currentRedactionLog.getRedactionLogEntry()); currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
Map<String, RedactionLogEntry> addedIds = new HashMap<>(); Map<String, RedactionLogEntry> addedIds = new HashMap<>();
added.forEach(entry -> { added.forEach(entry -> {
@ -78,16 +66,16 @@ public class RedactionChangeLogService {
newRedactionLogEntries.forEach(entry -> { newRedactionLogEntries.forEach(entry -> {
if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) { if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges(); List<Change> changes = entry.getChanges();
changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now())); changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
var newEntry = addedIds.get(entry.getId()); var newEntry = addedIds.get(entry.getId());
newEntry.setChanges(changes); newEntry.setChanges(changes);
addedIds.put(entry.getId(), newEntry); addedIds.put(entry.getId(), newEntry);
toRemove.add(entry); toRemove.add(entry);
} else if (removedIds.contains(entry.getId())) { } else if (removedIds.contains(entry.getId())) {
entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now())); entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
} else if (addedIds.containsKey(entry.getId())) { } else if (addedIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges(); List<Change> changes = entry.getChanges();
changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now())); changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
var newEntry = addedIds.get(entry.getId()); var newEntry = addedIds.get(entry.getId());
newEntry.setChanges(changes); newEntry.setChanges(changes);
addedIds.put(entry.getId(), newEntry); addedIds.put(entry.getId(), newEntry);
@ -98,8 +86,8 @@ public class RedactionChangeLogService {
newRedactionLogEntries.removeAll(toRemove); newRedactionLogEntries.removeAll(toRemove);
addedIds.forEach((k, v) -> { addedIds.forEach((k, v) -> {
if(v.getChanges().isEmpty()) { if (v.getChanges().isEmpty()) {
v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now()));
} }
newRedactionLogEntries.add(v); newRedactionLogEntries.add(v);
}); });