Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7384: migration fixes

Browse Source 
* index files from trash aswell
* merge manual local entries with existing ones during unprocessed changes merge
* don't add ManualRedactionEntries on local changes for Images
This commit is contained in:
Kilian Schuettler 2024-04-05 12:56:42 +02:00
parent 6525e0c2bc
commit 7df02d588a
3 changed files with 87 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -14,6 +14,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -86,7 +87,12 @@ public class EntityLogMergeService {
List<EntityLogEntry> entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry());
buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier).forEach(entityLogEntries::add);
Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions,
entityLog,
dossier).collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
entityLogEntries.addAll(addedLocalManualEntries.values());
buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add);
@ -95,6 +101,12 @@ public class EntityLogMergeService {
EntityLogEntry entityLogEntry = entityLogEntries.get(i + numberOfAddedPendingEntries);
if (isDuplicatedByAny(entityLogEntry, addedLocalManualEntries)) {
mergeOriginalAndLocallyAddedEntries(addedLocalManualEntries, entityLogEntry, analysisNumber);
continue;
}
if (allManualChanges.containsKey(entityLogEntry.getId())) {
List<EntityLogEntry> pendingImageRecategorizations = mergeLocalManualChangesAndReturnNonMergeableAsPending(dossier,
@ -121,6 +133,18 @@ public class EntityLogMergeService {
}
private static void mergeOriginalAndLocallyAddedEntries(Map<String, EntityLogEntry> addedLocalManualEntries, EntityLogEntry entityLogEntry, int analysisNumber) {
EntityLogEntry duplicateManualEntry = addedLocalManualEntries.get(entityLogEntry.getId());
duplicateManualEntry.getEngines().addAll(entityLogEntry.getEngines());
// Mark existing entry REMOVED, will be replaced by the manual one.
entityLogEntry.setState(EntryState.REMOVED);
entityLogEntry.setReason("Removed due to duplicate locally added manual entity.");
entityLogEntry.getChanges().add(Change.builder().type(ChangeType.REMOVED).dateTime(OffsetDateTime.now()).analysisNumber(analysisNumber).build());
}
private static List<EntityLogEntry> concatLists(List<EntityLogEntry> pendingDictionaryEntries, List<EntityLogEntry> pendingImageRecategorizations) {
return Stream.of(pendingDictionaryEntries, pendingImageRecategorizations)
@ -234,8 +258,7 @@ public class EntityLogMergeService {
.build());
List<Change> changes = new ArrayList<>();
changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED)
.build());
changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED).build());
boolean isHint = isHint(manualRedactionEntry.getType(), dossier);
@ -283,8 +306,7 @@ public class EntityLogMergeService {
existingEntry.setState(EntryState.REMOVED);
List<Change> falsePositiveChanges = new ArrayList<>();
falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED)
.build());
falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED).build());
if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) {
existingEntry.getChanges().addAll(falsePositiveChanges);
} else {
@ -306,7 +328,6 @@ public class EntityLogMergeService {
.userId(idRemoval.getUser())
.propertyChanges(Collections.emptyMap())
.build());
}
@ -327,16 +348,19 @@ public class EntityLogMergeService {
manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
}
entityLogEntry.getManualChanges().add(manualChange.build());
if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry())
&& !manualResizeRedaction.getUpdateDictionary()
&& !manualResizeRedaction.isAddToAllDossiers()) {
entityLogEntry.setState(EntryState.REMOVED);
}
}
private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
private boolean isDuplicatedByAny(EntityLogEntry entry, Map<String, EntityLogEntry> addedLocalManualEntryIds) {
return !entry.getEngines().contains(Engine.MANUAL) && addedLocalManualEntryIds.containsKey(entry.getId()) && !addedLocalManualEntryIds.get(entry.getId()).equals(entry);
}
@Deprecated(forRemoval = true)
private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange,
EntityLogEntry entityLogEntry,
int analysisNumber) {
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
@ -352,13 +376,10 @@ public class EntityLogMergeService {
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) {
entityLogEntry.setState(EntryState.REMOVED);
}
}
@Deprecated(forRemoval = true)
private Map<String, String> getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) {
Map<String, String> propertyChanges = new HashMap<>();
@ -375,7 +396,10 @@ public class EntityLogMergeService {
}
private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization,
EntityLogEntry entityLogEntry,
DossierEntity dossier,
int analysisNumber) {
if (entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT)) {
return pendingDictionaryEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
@ -417,10 +441,6 @@ public class EntityLogMergeService {
.userId(recategorization.getUser())
.propertyChanges(getPropertyChanges(recategorization))
.build());
if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) && !recategorization.isAddToDictionary() && !recategorization.isAddToAllDossiers()) {
entityLogEntry.setState(EntryState.REMOVED);
}
return null;
}
@ -459,10 +479,6 @@ public class EntityLogMergeService {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) {
entityLogEntry.setState(EntryState.REMOVED);
}
}
@ -479,11 +495,9 @@ public class EntityLogMergeService {
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) {
changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType)
.build());
changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType).build());
} else {
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType)
.build());
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
}
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/IndexingService.java
View File

@ -9,8 +9,6 @@ import org.apache.commons.lang3.tuple.Pair;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -25,7 +23,6 @@ import lombok.RequiredArgsConstructor;
public class IndexingService {
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final DossierService dossierService;
private final FileStatusPersistenceService fileStatusPersistenceService;
@ -53,7 +50,7 @@ public class IndexingService {
for (Pair<String, String> reindexDossierId : reindexDossierIds) {
List<FileEntity> fileStatuses = fileStatusPersistenceService.getStatusesForDossier(reindexDossierId.getRight());
for (FileEntity fileStatus : fileStatuses) {
if (fileStatus.isSoftOrHardDeleted()) {
if (fileStatus.getHardDeletedTime() != null) {
continue;
}
if (fileIds != null && !fileIds.isEmpty() && !fileIds.contains(fileStatus.getId())) {

85
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionMapper.java
View File

@ -59,7 +59,7 @@ public class ManualRedactionMapper {
}
public AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
private static AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
return AddRedactionRequest.builder()
.value(StringCleaningUtility.cleanString(addRedactionRequest.getValue()))
@ -129,7 +129,7 @@ public class ManualRedactionMapper {
.comment(forceRedactionRequestModel.getComment())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -158,7 +158,7 @@ public class ManualRedactionMapper {
.value(legalBasisChangeRequest.getValue())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -198,7 +198,10 @@ public class ManualRedactionMapper {
.section(recategorizationRequest.getSection())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !recategorizationRequest.isAddToAllDossiers() && !recategorizationRequest.isAddToDictionary()) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
&& !recategorizationRequest.isAddToAllDossiers()
&& !recategorizationRequest.isAddToDictionary()
&& entryIsEntityType(entityLogEntry)) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -230,7 +233,7 @@ public class ManualRedactionMapper {
.addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -241,28 +244,6 @@ public class ManualRedactionMapper {
}
private EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
return entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log."));
}
private DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
if (entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) && entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
} else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_POSITIVE;
} else {
return DictionaryEntryType.ENTRY;
}
}
private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
@ -279,7 +260,7 @@ public class ManualRedactionMapper {
.legalBasis(entityLogEntry.getLegalBasis())
.textAfter(entityLogEntry.getTextAfter())
.textBefore(entityLogEntry.getTextBefore())
.dictionaryEntryType(convertEntryType(entityLogEntry))
.dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
.fileId(fileId)
.requestDate(OffsetDateTime.now())
.build();
@ -288,25 +269,43 @@ public class ManualRedactionMapper {
}
private List<Rectangle> convertPositions(List<Position> positions) {
private static boolean entryIsEntityType(EntityLogEntry entityLogEntry) {
return entityLogEntry.getEntryType().equals(EntryType.ENTITY) //
|| entityLogEntry.getEntryType().equals(EntryType.HINT) //
|| entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) //
|| entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION) //
|| entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE);
}
private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
return entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log."));
}
private static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
} else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_POSITIVE;
} else {
return DictionaryEntryType.ENTRY;
}
}
private static List<Rectangle> convertPositions(List<Position> positions) {
return positions.stream()
.map(rectangle -> new Rectangle(rectangle.x(), rectangle.y(), rectangle.w(), rectangle.h(), rectangle.getPageNumber()))
.toList();
}
private DictionaryEntryType convertEntryType(EntityLogEntry entityLogEntry) {
if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_POSITIVE;
}
if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
}
return DictionaryEntryType.ENTRY;
}
}