Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #428: RED-4064: Always trigger reanalysis instead of surrounding text analysis when add a resize redaction

Browse Source 
Merge in RED/persistence-service from RED-4064 to master

* commit '6ad0f804dc209d8f741755756ac01c7b8b050cdf':
  RED-4064: Always trigger reanalysis instead of surrounding text analysis when add a resize redaction
This commit is contained in:
Dominique Eiflaender 2022-06-03 11:03:57 +02:00
commit 1f9de41126
1 changed files with 35 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -26,7 +26,6 @@ import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualResizeRedactionMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
@ -62,7 +61,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Manual
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ResizeRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@ -104,7 +102,8 @@ public class ManualRedactionService {
private final HashFunction hashFunction = Hashing.murmur3_128();
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId, List<AddRedactionRequest> addRedactionRequests) {
public List<ManualAddResponse> addAddRedaction(String dossierId, String fileId,
List<AddRedactionRequest> addRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
@ -117,7 +116,8 @@ public class ManualRedactionService {
}
validatePositions(fileId, addRedactionRequest);
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8).toString();
String annotationId = hashFunction.hashString(fileId + addRedactionRequest, StandardCharsets.UTF_8)
.toString();
addRedactionPersistenceService.insert(fileId, annotationId, addRedactionRequest);
Long commentId = null;
@ -132,13 +132,12 @@ public class ManualRedactionService {
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId);
}else {
} else {
var manualTextRedactions = convert(response.stream()
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
@ -146,7 +145,9 @@ public class ManualRedactionService {
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
ManualRedactions manualRedactions = ManualRedactions.builder()
.entriesToAdd(new HashSet<>(manualTextRedactions))
.build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
@ -155,7 +156,9 @@ public class ManualRedactionService {
return response;
}
private void validateDictionary(AddRedactionRequest addRedactionRequest) {
try {
if (!addRedactionRequest.isForceAddToDictionary() && stopwordService.isStopword(addRedactionRequest.getValue())) {
throw new ConflictException("The entry you are trying to add is a stopword");
@ -166,14 +169,20 @@ public class ManualRedactionService {
}
}
private void validatePositions(String fileId, AddRedactionRequest addRedactionRequest) {
var numberOfPages = fileStatusService.getStatus(fileId).getNumberOfPages();
addRedactionRequest.getPositions().stream().filter(p -> p.getPage() > numberOfPages).findAny()
addRedactionRequest.getPositions()
.stream()
.filter(p -> p.getPage() > numberOfPages)
.findAny()
.ifPresent((p) -> new BadRequestException("Invalid page found in the request"));
}
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId,
List<RemoveRedactionRequest> removeRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
@ -202,14 +211,18 @@ public class ManualRedactionService {
.stream()
.filter(entry -> entry.getId().equals(removeRedactionRequest.getAnnotationId()))
.findFirst();
actionPerformed = actionPerformed || redactionLogEntryOptional.isPresent() && redactionLogEntryOptional.get().isHint();
actionPerformed = actionPerformed || redactionLogEntryOptional.isPresent() && redactionLogEntryOptional.get()
.isHint();
}
var removedFromDictionary = handleRemoveFromDictionary(redactionLog, dossier, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), false);
actionPerformed = actionPerformed || removedFromDictionary;
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build());
}
}
@ -475,7 +488,6 @@ public class ManualRedactionService {
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
boolean reprocess = false;
RedactionLog redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true, true);
for (ResizeRedactionRequest resizeRedactionRequest : resizeRedactionRequests) {
@ -487,18 +499,19 @@ public class ManualRedactionService {
.stream()
.filter(r -> r.getId().equals(resizeRedactionRequest.getAnnotationId()))
.findFirst();
if (redactionLogEntry.isPresent() && (redactionLogEntry.get().isDictionaryEntry() || redactionLogEntry.get().isDossierDictionaryEntry())) {
if (redactionLogEntry.isPresent() && (redactionLogEntry.get()
.isDictionaryEntry() || redactionLogEntry.get().isDossierDictionaryEntry())) {
String typeId = redactionLogEntry.get().getType() + ":" + dossier.getDossierTemplateId();
if (redactionLogEntry.get().getValue().length() > resizeRedactionRequest.getValue().length()) {
log.info("Remove old value '{}' from dictionary", redactionLogEntry.get().getValue());
entryPersistenceService.deleteEntries(typeId, List.of(redactionLogEntry.get().getValue()), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY);
entryPersistenceService.deleteEntries(typeId, List.of(redactionLogEntry.get()
.getValue()), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY);
}
log.info("Add new value '{}' to dictionary", resizeRedactionRequest.getValue());
entryPersistenceService.addEntries(typeId, new HashSet<>(Collections.singleton(resizeRedactionRequest.getValue())), redactionLog.getAnalysisVersion(), DictionaryEntryType.ENTRY);
reprocess = true;
}
}
@ -507,16 +520,14 @@ public class ManualRedactionService {
commentId = addComment(fileId, resizeRedactionRequest.getAnnotationId(), resizeRedactionRequest.getComment(), resizeRedactionRequest.getUser()).getId();
}
var loaded = convert(getResizeRedaction(fileId, resizeRedactionRequest.getAnnotationId()), ManualResizeRedaction.class, new ManualResizeRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder()
.annotationId(resizeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build());
}
if (reprocess) {
reprocess(dossierId, fileId);
}
reprocess(dossierId, fileId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
@ -545,7 +556,6 @@ public class ManualRedactionService {
if (idRemoval.isRemoveFromDictionary()) {
var redactionLogEntry = redactionLogEntryOptional.get();
if (annotationStatus == AnnotationStatus.APPROVED) {
@ -627,8 +637,6 @@ public class ManualRedactionService {
// These are marked as processed once surrounding text is computed ( TBD if this is correct ? )
manualRedactions.getEntriesToAdd()
.forEach(e -> addRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
manualRedactions.getResizeRedactions()
.forEach(e -> resizeRedactionPersistenceService.updateSurroundingText(new AnnotationEntityId(e.getAnnotationId(), fileId), e.getTextBefore(), e.getTextAfter()));
}
@ -794,7 +802,8 @@ public class ManualRedactionService {
}
private void addToDictionary(String typeId, String value, String dossierId, String fileId, DictionaryEntryType dictionaryEntryType) {
private void addToDictionary(String typeId, String value, String dossierId, String fileId,
DictionaryEntryType dictionaryEntryType) {
try {
log.debug("Adding entry: {} to {} for {} / {}", value, typeId, dossierId, fileId);