Merge branch 'RED-7384-fp' into 'master'

RED-7384: migration fixes

Closes RED-7384

See merge request redactmanager/persistence-service!430
This commit is contained in:
Kilian Schüttler 2024-04-05 13:50:14 +02:00
commit d7e5e16351
4 changed files with 104 additions and 89 deletions

View File

@ -14,6 +14,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -86,7 +87,12 @@ public class EntityLogMergeService {
List<EntityLogEntry> entityLogEntries = new LinkedList<>(entityLog.getEntityLogEntry());
buildUnprocessedLocalManualRedactions(unprocessedManualRedactions, entityLog, dossier).forEach(entityLogEntries::add);
Map<String, EntityLogEntry> addedLocalManualEntries = buildUnprocessedLocalManualRedactions(unprocessedManualRedactions,
entityLog,
dossier).collect(Collectors.toMap(EntityLogEntry::getId, Function.identity()));
entityLogEntries.addAll(addedLocalManualEntries.values());
buildPendingDictionaryChanges(unprocessedManualRedactions).forEach(entityLogEntries::add);
@ -95,6 +101,12 @@ public class EntityLogMergeService {
EntityLogEntry entityLogEntry = entityLogEntries.get(i + numberOfAddedPendingEntries);
if (isDuplicatedByAny(entityLogEntry, addedLocalManualEntries)) {
mergeOriginalAndLocallyAddedEntries(addedLocalManualEntries, entityLogEntry, analysisNumber);
continue;
}
if (allManualChanges.containsKey(entityLogEntry.getId())) {
List<EntityLogEntry> pendingImageRecategorizations = mergeLocalManualChangesAndReturnNonMergeableAsPending(dossier,
@ -121,6 +133,18 @@ public class EntityLogMergeService {
}
private static void mergeOriginalAndLocallyAddedEntries(Map<String, EntityLogEntry> addedLocalManualEntries, EntityLogEntry entityLogEntry, int analysisNumber) {
EntityLogEntry duplicateManualEntry = addedLocalManualEntries.get(entityLogEntry.getId());
duplicateManualEntry.getEngines().addAll(entityLogEntry.getEngines());
// Mark existing entry REMOVED, will be replaced by the manual one.
entityLogEntry.setState(EntryState.REMOVED);
entityLogEntry.setReason("Removed due to duplicate locally added manual entity.");
entityLogEntry.getChanges().add(Change.builder().type(ChangeType.REMOVED).dateTime(OffsetDateTime.now()).analysisNumber(analysisNumber).build());
}
private static List<EntityLogEntry> concatLists(List<EntityLogEntry> pendingDictionaryEntries, List<EntityLogEntry> pendingImageRecategorizations) {
return Stream.of(pendingDictionaryEntries, pendingImageRecategorizations)
@ -234,8 +258,7 @@ public class EntityLogMergeService {
.build());
List<Change> changes = new ArrayList<>();
changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED)
.build());
changes.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED).build());
boolean isHint = isHint(manualRedactionEntry.getType(), dossier);
@ -283,8 +306,7 @@ public class EntityLogMergeService {
existingEntry.setState(EntryState.REMOVED);
List<Change> falsePositiveChanges = new ArrayList<>();
falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED)
.build());
falsePositiveChanges.add(Change.builder().analysisNumber(entityLog.getAnalysisNumber()).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED).build());
if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) {
existingEntry.getChanges().addAll(falsePositiveChanges);
} else {
@ -306,7 +328,6 @@ public class EntityLogMergeService {
.userId(idRemoval.getUser())
.propertyChanges(Collections.emptyMap())
.build());
}
@ -327,16 +348,19 @@ public class EntityLogMergeService {
manualChange.propertyChanges(Map.of("value", manualResizeRedaction.getValue()));
}
entityLogEntry.getManualChanges().add(manualChange.build());
if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry())
&& !manualResizeRedaction.getUpdateDictionary()
&& !manualResizeRedaction.isAddToAllDossiers()) {
entityLogEntry.setState(EntryState.REMOVED);
}
}
private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
private boolean isDuplicatedByAny(EntityLogEntry entry, Map<String, EntityLogEntry> addedLocalManualEntryIds) {
return !entry.getEngines().contains(Engine.MANUAL) && addedLocalManualEntryIds.containsKey(entry.getId()) && !addedLocalManualEntryIds.get(entry.getId()).equals(entry);
}
@Deprecated(forRemoval = true)
private void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange,
EntityLogEntry entityLogEntry,
int analysisNumber) {
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
@ -352,13 +376,10 @@ public class EntityLogMergeService {
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) {
entityLogEntry.setState(EntryState.REMOVED);
}
}
@Deprecated(forRemoval = true)
private Map<String, String> getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) {
Map<String, String> propertyChanges = new HashMap<>();
@ -375,7 +396,10 @@ public class EntityLogMergeService {
}
private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
private EntityLogEntry mergeRecategorization(ManualRecategorization recategorization,
EntityLogEntry entityLogEntry,
DossierEntity dossier,
int analysisNumber) {
if (entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT)) {
return pendingDictionaryEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
@ -417,10 +441,6 @@ public class EntityLogMergeService {
.userId(recategorization.getUser())
.propertyChanges(getPropertyChanges(recategorization))
.build());
if ((entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) && !recategorization.isAddToDictionary() && !recategorization.isAddToAllDossiers()) {
entityLogEntry.setState(EntryState.REMOVED);
}
return null;
}
@ -459,10 +479,6 @@ public class EntityLogMergeService {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
if (entityLogEntry.isDictionaryEntry() || entityLogEntry.isDossierDictionaryEntry()) {
entityLogEntry.setState(EntryState.REMOVED);
}
}
@ -479,11 +495,9 @@ public class EntityLogMergeService {
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) {
changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType)
.build());
changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType).build());
} else {
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType)
.build());
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
}
}

View File

@ -9,8 +9,6 @@ import org.apache.commons.lang3.tuple.Pair;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -25,7 +23,6 @@ import lombok.RequiredArgsConstructor;
public class IndexingService {
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final DossierService dossierService;
private final FileStatusPersistenceService fileStatusPersistenceService;
@ -53,7 +50,7 @@ public class IndexingService {
for (Pair<String, String> reindexDossierId : reindexDossierIds) {
List<FileEntity> fileStatuses = fileStatusPersistenceService.getStatusesForDossier(reindexDossierId.getRight());
for (FileEntity fileStatus : fileStatuses) {
if (fileStatus.isSoftOrHardDeleted()) {
if (fileStatus.getHardDeletedTime() != null) {
continue;
}
if (fileIds != null && !fileIds.isEmpty() && !fileIds.contains(fileStatus.getId())) {

View File

@ -59,7 +59,7 @@ public class ManualRedactionMapper {
}
public AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
private static AddRedactionRequest toAddRedactionRequest(String dossierId, String dossierTemplateId, AddRedactionRequestModel addRedactionRequest) {
return AddRedactionRequest.builder()
.value(StringCleaningUtility.cleanString(addRedactionRequest.getValue()))
@ -129,7 +129,7 @@ public class ManualRedactionMapper {
.comment(forceRedactionRequestModel.getComment())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -158,7 +158,7 @@ public class ManualRedactionMapper {
.value(legalBasisChangeRequest.getValue())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry)) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -198,7 +198,10 @@ public class ManualRedactionMapper {
.section(recategorizationRequest.getSection())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !recategorizationRequest.isAddToAllDossiers() && !recategorizationRequest.isAddToDictionary()) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL)
&& !recategorizationRequest.isAddToAllDossiers()
&& !recategorizationRequest.isAddToDictionary()
&& entryIsEntityType(entityLogEntry)) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -230,7 +233,7 @@ public class ManualRedactionMapper {
.addToAllDossiers(resizeRedactionRequest.isAddToAllDossiers())
.build();
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
if (!entityLogEntry.getEngines().contains(Engine.MANUAL) && entryIsEntityType(entityLogEntry) && !request.isAddToAllDossiers() && !request.getUpdateDictionary()) {
addManualRedactionEntry(fileId, entityLogEntry);
}
@ -241,28 +244,6 @@ public class ManualRedactionMapper {
}
private EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
return entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log."));
}
private DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
if (entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) && entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
} else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_POSITIVE;
} else {
return DictionaryEntryType.ENTRY;
}
}
private void addManualRedactionEntry(String fileId, EntityLogEntry entityLogEntry) {
ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
@ -279,7 +260,7 @@ public class ManualRedactionMapper {
.legalBasis(entityLogEntry.getLegalBasis())
.textAfter(entityLogEntry.getTextAfter())
.textBefore(entityLogEntry.getTextBefore())
.dictionaryEntryType(convertEntryType(entityLogEntry))
.dictionaryEntryType(getDictionaryEntryType(entityLogEntry))
.fileId(fileId)
.requestDate(OffsetDateTime.now())
.build();
@ -288,25 +269,43 @@ public class ManualRedactionMapper {
}
private List<Rectangle> convertPositions(List<Position> positions) {
private static boolean entryIsEntityType(EntityLogEntry entityLogEntry) {
return entityLogEntry.getEntryType().equals(EntryType.ENTITY) //
|| entityLogEntry.getEntryType().equals(EntryType.HINT) //
|| entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION) //
|| entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION) //
|| entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE);
}
private static EntityLogEntry getEntityLogEntry(EntityLog entityLog, String annotationId) {
return entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow(() -> new NotFoundException("Annotation does not exist in entity log."));
}
private static DictionaryEntryType getDictionaryEntryType(EntityLogEntry entityLogEntry) {
if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
} else if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_POSITIVE;
} else {
return DictionaryEntryType.ENTRY;
}
}
private static List<Rectangle> convertPositions(List<Position> positions) {
return positions.stream()
.map(rectangle -> new Rectangle(rectangle.x(), rectangle.y(), rectangle.w(), rectangle.h(), rectangle.getPageNumber()))
.toList();
}
private DictionaryEntryType convertEntryType(EntityLogEntry entityLogEntry) {
if (entityLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)) {
return DictionaryEntryType.FALSE_POSITIVE;
}
if (entityLogEntry.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) {
return DictionaryEntryType.FALSE_RECOMMENDATION;
}
return DictionaryEntryType.ENTRY;
}
}

View File

@ -243,14 +243,13 @@ public class EntityLogMergeTest {
String dossierId = "dossierId";
String dossierTemplateId = "dossierTemplateId";
String entryToAddId = UUID.randomUUID().toString();
String rectangleToAddId = UUID.randomUUID().toString();
String entryToRemoveId = UUID.randomUUID().toString();
String entryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
ManualRedactions manualRedactions = provideManualRedactions(entryLegalBasisId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, true);
@ -272,21 +271,22 @@ public class EntityLogMergeTest {
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
.findFirst();
assertTrue(optionalResizeEntryLogEntry.isPresent());
assertEquals(optionalResizeEntryLogEntry.get().getEntryType(), EntryType.ENTITY);
assertEquals(EntryType.ENTITY, optionalResizeEntryLogEntry.get().getEntryType());
var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
var legalBasisEntries = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId))
.findFirst();
assertTrue(optionalLegalBasisEntryLogEntry.isPresent());
assertEquals(optionalLegalBasisEntryLogEntry.get().getState(), EntryState.REMOVED);
.toList();
assertEquals(2, legalBasisEntries.size());
assertEquals(EntryState.REMOVED, legalBasisEntries.get(0).getState());
assertEquals(EntryState.APPLIED, legalBasisEntries.get(1).getState());
var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId))
.findFirst();
assertTrue(optionalForceRedactionEntryLogEntry.isPresent());
assertEquals(optionalForceRedactionEntryLogEntry.get().getState(), EntryState.REMOVED);
assertEquals(EntryState.APPLIED, optionalForceRedactionEntryLogEntry.get().getState());
}
@ -302,7 +302,6 @@ public class EntityLogMergeTest {
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = new ManualRedactions();
List<Rectangle> positions = new ArrayList<>();
positions.add(new Rectangle(2, 2, 2, 2, 1));
@ -329,10 +328,16 @@ public class EntityLogMergeTest {
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
var resizedEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)).findFirst().get();
var resizedEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
.findFirst()
.get();
int index = response.getEntityLogEntry().indexOf(resizedEntry);
assertEquals(response.getEntityLogEntry().get(index + 1).getId(), resizedEntry.getId());
assertEquals(response.getEntityLogEntry().get(index + 1).getState(), EntryState.PENDING);
assertEquals(response.getEntityLogEntry()
.get(index + 1).getId(), resizedEntry.getId());
assertEquals(response.getEntityLogEntry()
.get(index + 1).getState(), EntryState.PENDING);
}