Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #318: analysis version in redactionlog and changes

Browse Source 
Merge in RED/redaction-service from analysis-number-in-redaction-log-m to master

* commit '1cad386db83a2d2057701989c0c179b770248f5e':
  analysis version in redactionlog and changes
This commit is contained in:
Timo Bejan 2022-01-13 09:23:34 +01:00
commit f05977b7ae
6 changed files with 51 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -26,6 +26,7 @@ public class AnalyzeRequest {
private String dossierTemplateId;
private ManualRedactions manualRedactions;
private OffsetDateTime lastProcessed;
private int analysisNumber;
@Builder.Default
private Set<Integer> excludedPages = new HashSet<>();

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java
View File

@ -28,6 +28,7 @@ public class AnalyzeResult {
private boolean wasReanalyzed;
private int analysisVersion;
private int analysisNumber;
private ManualRedactions manualRedactions;

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java
View File

@ -13,6 +13,7 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class Change {
private int analysisNumber;
private ChangeType type;
private OffsetDateTime dateTime;
}

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java
View File

@ -18,6 +18,11 @@ public class RedactionLog {
*/
private long analysisVersion;
/**
* Which analysis created this redactionLog.
*/
private int analysisNumber;
private List<RedactionLogEntry> redactionLogEntry;
private List<LegalBasis> legalBasis;
@ -26,4 +31,5 @@ public class RedactionLog {
private long rulesVersion = -1;
private long legalBasisVersion = -1;
}

60
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -1,42 +1,31 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Service
@ -71,7 +60,7 @@ public class AnalyzeService {
.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
Map<Integer, List<PdfImage>> pdfImages = null;
if(redactionServiceSettings.isEnableImageClassification()) {
if (redactionServiceSettings.isEnableImageClassification()) {
pdfImages = imageService.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
}
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, pdfImages);
@ -83,7 +72,7 @@ public class AnalyzeService {
List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
Text text = new Text(pageCount,sectionTexts);
Text text = new Text(pageCount, sectionTexts);
// enhance section grid with headline data
sectionTexts.forEach(sectionText -> classifiedDoc
@ -106,7 +95,7 @@ public class AnalyzeService {
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
}
@ -125,10 +114,9 @@ public class AnalyzeService {
.getDossierTemplateId());
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary
.getVersion()
.getDossierTemplateVersion(), dictionary.getVersion()
.getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
}
@ -158,7 +146,7 @@ public class AnalyzeService {
}
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
}
@ -223,7 +211,8 @@ public class AnalyzeService {
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
redactionLog, analyzeRequest.getAnalysisNumber());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
.getRedactionLog());
@ -236,6 +225,7 @@ public class AnalyzeService {
.numberOfPages(text.getNumberOfPages())
.hasUpdates(redactionLogChange.isHasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(redactionLog.getRulesVersion())
.dictionaryVersion(redactionLog.getDictionaryVersion())
.legalBasisVersion(redactionLog.getLegalBasisVersion())
@ -252,12 +242,12 @@ public class AnalyzeService {
}
return Stream.concat(manualRedactions.getLegalBasisChanges()
.stream()
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
.stream()
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
.stream()
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
.stream()
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
.stream()
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
.stream()
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
.collect(Collectors.toSet());
}

46
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java
View File

@ -1,25 +1,14 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Change;
import com.iqser.red.service.redaction.v1.model.ChangeType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogChanges;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Service
@ -29,7 +18,7 @@ public class RedactionChangeLogService {
private final RedactionStorageService redactionStorageService;
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) {
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
long start = System.currentTimeMillis();
@ -37,7 +26,7 @@ public class RedactionChangeLogService {
if (previousRedactionLog == null) {
currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
});
return new RedactionLogChanges(currentRedactionLog, false);
}
@ -50,17 +39,16 @@ public class RedactionChangeLogService {
.equals(ChangeType.REMOVED))
.collect(Collectors.toList());
Set<RedactionLogEntry> added = new HashSet<>(currentRedactionLog.getRedactionLogEntry().stream()
Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
.filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges()
.get(entry.getChanges().size() - 1)
.getType()
.equals(ChangeType.REMOVED))
.collect(Collectors.toList()));
.equals(ChangeType.REMOVED)).collect(Collectors.toSet());
added.removeAll(notRemovedPreviousEntries);
notRemovedPreviousEntries.forEach(added::remove);
Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
removed.removeAll(currentRedactionLog.getRedactionLogEntry());
currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
Map<String, RedactionLogEntry> addedIds = new HashMap<>();
added.forEach(entry -> {
@ -78,16 +66,16 @@ public class RedactionChangeLogService {
newRedactionLogEntries.forEach(entry -> {
if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges();
changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now()));
changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
var newEntry = addedIds.get(entry.getId());
newEntry.setChanges(changes);
addedIds.put(entry.getId(), newEntry);
toRemove.add(entry);
} else if (removedIds.contains(entry.getId())) {
entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now()));
entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
} else if (addedIds.containsKey(entry.getId())) {
List<Change> changes = entry.getChanges();
changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
var newEntry = addedIds.get(entry.getId());
newEntry.setChanges(changes);
addedIds.put(entry.getId(), newEntry);
@ -98,8 +86,8 @@ public class RedactionChangeLogService {
newRedactionLogEntries.removeAll(toRemove);
addedIds.forEach((k, v) -> {
if(v.getChanges().isEmpty()) {
v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
if (v.getChanges().isEmpty()) {
v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now()));
}
newRedactionLogEntries.add(v);
});