Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-4064: Apply all resize redactions directly after find, calculate surrounding text in renalysis

Browse Source
This commit is contained in:
deiflaender 2022-06-03 10:51:21 +02:00
parent a16f9c6d6e
commit 3f76401c7d
7 changed files with 101 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -86,7 +86,7 @@ public class Section {
.collect(Collectors.toSet());
List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
Set<Entity> finalResult = new HashSet<>();
@ -886,30 +886,11 @@ public class Section {
}
});
return applyResizeRedactions(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary));
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
}
private Set<Entity> applyResizeRedactions(Set<Entity> entitiesWithPositions) {
if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){
return entitiesWithPositions;
}
entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> {
manualRedactions.getResizeRedactions().forEach(resize -> {
if (resize.getAnnotationId().equals(pos.getId()) && resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
int start = e.getWord().indexOf(resize.getValue());
e.setStart(e.getStart() + start);
e.setEnd(e.getStart() + resize.getValue().length());
e.setResized(true);
e.setWord(resize.getValue());
}
});
}));
return entitiesWithPositions;
}
private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {
@ -963,9 +944,7 @@ public class Section {
Set<Entity> singleEntitySet = new HashSet<>();
singleEntitySet.add(entity);
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary);
applyResizeRedactions(singleEntitySet);
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
@ -9,5 +11,6 @@ public class SectionSearchableTextPair {
private Section section;
private SearchableText searchableText;
private List<Integer> cellStarts;
}

53
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -78,8 +78,7 @@ public class EntityRedactionService {
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
for (SectionText reanalysisSection : reanalysisSections) {
Entities entities = findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection
.getSectionNumber(), dictionary, local, nerEntities, reanalysisSection.getCellStarts());
Entities entities = findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection.getSectionNumber(), dictionary, local, nerEntities, reanalysisSection.getCellStarts(), analyzeRequest.getManualRedactions());
if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection
@ -88,20 +87,26 @@ public class EntityRedactionService {
surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
}
if (!local && analyzeRequest.getManualRedactions() != null) {
var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
var approvedForceRedactions = analyzeRequest.getManualRedactions()
.getForceRedactions()
.stream()
.filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
.filter(fr -> fr.getRequestDate() != null)
.collect(Collectors.toList());
// only approved id removals, that haven't been forced back afterwards
var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
var idsToRemove = analyzeRequest.getManualRedactions()
.getIdsToRemove()
.stream()
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
.filter(idr -> idr.getRequestDate() != null)
.filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
.filter(idr -> approvedForceRedactions.stream()
.noneMatch(forceRedact -> forceRedact.getAnnotationId()
.equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
.isAfter(idr.getRequestDate())))
.map(IdRemoval::getAnnotationId)
.collect(Collectors.toSet());
if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
.isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
@ -133,9 +138,9 @@ public class EntityRedactionService {
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
.isLocal(false)
.dictionaryTypes(dictionary.getTypes())
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream
.concat(entities.getEntities().stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber())
.stream()).collect(Collectors.toSet()) : entities.getEntities())
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
.stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream())
.collect(Collectors.toSet()) : entities.getEntities())
.nerEntities(entities.getNerEntities())
.text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
.searchText(reanalysisSection.getSearchableText().toString())
@ -148,7 +153,7 @@ public class EntityRedactionService {
.sectionAreas(reanalysisSection.getSectionAreas())
.fileAttributes(analyzeRequest.getFileAttributes())
.manualRedactions(analyzeRequest.getManualRedactions())
.build(), reanalysisSection.getSearchableText()));
.build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
}
@ -156,6 +161,16 @@ public class EntityRedactionService {
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
var entriesWithoutSurroundingText = analysedSection.getEntities().stream().filter(e -> e.getTextAfter() == null && e.getTextBefore() == null).collect(Collectors.toSet());
if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts()
.isEmpty()) {
surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary, sectionSearchableTextPair.getCellStarts());
} else {
surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
}
entities.addAll(analysedSection.getEntities());
if (!local) {
@ -164,6 +179,7 @@ public class EntityRedactionService {
}
addLocalValuesToDictionary(analysedSection, dictionary);
}
});
return entities;
@ -182,11 +198,7 @@ public class EntityRedactionService {
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity
.getReferences(), entity.getEntityType()));
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity.getReferences(), entity.getEntityType()));
}
}
return entitiesPerPage;
@ -226,7 +238,7 @@ public class EntityRedactionService {
private Entities findEntities(SearchableText searchableText, String headline, int sectionNumber,
Dictionary dictionary, boolean local, NerEntities nerEntities,
List<Integer> cellStarts) {
List<Integer> cellStarts, ManualRedactions manualRedactions) {
Set<Entity> found = new HashSet<>();
String searchableString = searchableText.asString();
@ -239,8 +251,7 @@ public class EntityRedactionService {
for (DictionaryModel model : dictionary.getDictionaryModels()) {
var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
searchImplementation, model, new FindEntityDetails(model.getType(),headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, local? EntityType.RECOMMENDATION: EntityType.ENTITY));
var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString, searchImplementation, model, new FindEntityDetails(model.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
EntitySearchUtils.addOrAddEngine(found, entities);
}
@ -250,7 +261,7 @@ public class EntityRedactionService {
nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
}
return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions), nerFound);
}

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -1,9 +1,15 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.model.SectionArea;
@ -17,14 +23,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@Slf4j
@Service
@ -40,11 +41,10 @@ public class ManualRedactionSurroundingTextService {
long startTime = System.currentTimeMillis();
Text text = redactionStorageService.getText(dossierId, fileId);
List<ManualRedactionEntry> processedAddRedactions = new ArrayList<>();
List<ManualResizeRedaction> processedResizeRedactions = new ArrayList<>();
for (SectionText sectionText : text.getSectionTexts()) {
if (manualRedactions.getEntriesToAdd().isEmpty() && manualRedactions.getResizeRedactions().isEmpty()) {
if (manualRedactions.getEntriesToAdd().isEmpty()) {
break;
}
@ -61,23 +61,10 @@ public class ManualRedactionSurroundingTextService {
addItty.remove();
}
}
var resizeItty = manualRedactions.getResizeRedactions().iterator();
while (resizeItty.hasNext()) {
var manualResizeRedaction = resizeItty.next();
if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
manualResizeRedaction.setTextBefore(surroundingText.getLeft());
manualResizeRedaction.setTextAfter(surroundingText.getRight());
processedResizeRedactions.add(manualResizeRedaction);
resizeItty.remove();
}
}
}
}
manualRedactions.getEntriesToAdd().addAll(processedAddRedactions);
manualRedactions.getResizeRedactions().addAll(processedResizeRedactions);
return AnalyzeResult.builder()
.dossierId(dossierId)
@ -88,11 +75,11 @@ public class ManualRedactionSurroundingTextService {
}
private Pair<String, String> findSurroundingText(SectionText sectionText, String value, List<Rectangle> toFindPositions) {
private Pair<String, String> findSurroundingText(SectionText sectionText, String value,
List<Rectangle> toFindPositions) {
Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), new SearchImplementation(value, false),
new FindEntityDetails("dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, EntityType.ENTITY));
Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);
Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), new SearchImplementation(value, false), new FindEntityDetails("dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, EntityType.ENTITY));
Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null, null);
Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -290,15 +290,23 @@ public class RedactionLogMergeService {
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setValue(manualResizeRedact.getValue());
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
}
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
}
}
redactionLogEntry.setReason(manualOverrideReason);

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import lombok.experimental.UtilityClass;
@ -55,7 +56,7 @@ public class EntitySearchUtils {
}
public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary, ManualRedactions manualRedactions) {
Map<String, List<Entity>> entitiesByWord = new HashMap<>();
@ -79,12 +80,46 @@ public class EntitySearchUtils {
}
}
if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()){
applyResizeRedactions(entities, manualRedactions);
}
removeEntitiesContainedInLarger(entities);
return entities;
}
private Set<Entity> applyResizeRedactions(Set<Entity> entitiesWithPositions, ManualRedactions manualRedactions) {
if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){
return entitiesWithPositions;
}
entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> {
manualRedactions.getResizeRedactions().forEach(resize -> {
if (resize.getAnnotationId().equals(pos.getId())) {
if (resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
int start = e.getWord().indexOf(resize.getValue());
e.setStart(e.getStart() + start);
e.setEnd(e.getStart() + resize.getValue().length());
e.setResized(true);
e.setWord(resize.getValue());
} else if(resize.getValue().length() > e.getWord().length() && resize.getValue().contains(e.getWord())){
int start = resize.getValue().indexOf(e.getWord());
e.setStart(e.getStart() - start);
e.setEnd(e.getStart() + resize.getValue().length());
e.setResized(true);
e.setWord(resize.getValue());
}
}
});
}));
return entitiesWithPositions;
}
public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
List<Entity> wordsToRemove = new ArrayList<>();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -845,6 +845,8 @@ public class RedactionIntegrationTest {
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions)
.colors(colors)