Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-7384-bp: migration fixes

Browse Source
This commit is contained in:
Kilian Schüttler 2024-03-08 14:41:01 +01:00 committed by Dominique Eifländer
parent ba574e8d15
commit bfee997c22
9 changed files with 193 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/MigrationStatusController.java vendored
View File

@ -1,5 +1,6 @@
package com.iqser.red.persistence.service.v1.external.api.impl.controller;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.SaasMigrationStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.migration.SaasMigrationService;
@ -18,6 +19,7 @@ import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.SaasMigrationStatus.*;
@ -47,10 +49,8 @@ public class MigrationStatusController implements MigrationStatusResource {
var filesInErrorState = saasMigrationStatusPersistenceService.findAllByStatus(ERROR);
Map<String, String> errorCauses = new HashMap<>();
filesInErrorState.forEach(errorFile -> {
errorCauses.put(errorFile.getFileId(), errorFile.getErrorCause());
});
var errorCauses = filesInErrorState.stream()
.collect(Collectors.toMap(errorFile -> errorFile.getDossierId() + "/" + errorFile.getFileId(), SaasMigrationStatusEntity::getErrorCause));
return MigrationStatusResponse.builder().numberOfFilesToMigrate(numberOfFilesToMigrate).filesInStatus(filesInStatus).errorCauses(errorCauses).build();
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -92,14 +92,17 @@ public class SaasMigrationService implements TenantSyncService {
.stream()
.filter(dossier -> dossier.getHardDeletedTime() == null)
.toList();
for (var dossier : dossiers) {
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId())
.stream()
.filter(file -> file.getHardDeletedTime() == null)
.toList();
var migrationStati = saasMigrationStatusPersistenceService.findAll()
.stream()
.collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus));
for (var file : files) {
if (notExistsOrError(file, migrationStati)) {
// delete NER_ENTITIES since offsets depend on old document structure.

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java
View File

@ -24,10 +24,10 @@ import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Slf4j
public class EntityLogService {
FileManagementStorageService fileManagementStorageService;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/ManualRedactionService.java
View File

@ -454,7 +454,7 @@ public class ManualRedactionService {
.stream()
.map(p -> MagicConverter.convert(p, RectangleEntity.class))
.toList());
manualRedactionEntryEntity.setRequestDate(file.getAdded());
manualRedactionEntryEntity.setRequestDate(manualRedactionEntry.getRequestDate());
manualRedactionEntryEntity.setFileStatus(file);
return manualRedactionEntryEntity;
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/PendingDictionaryEntryFactory.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.manual
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -19,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@Service
public class PendingDictionaryEntryFactory {
@ -36,7 +38,8 @@ public class PendingDictionaryEntryFactory {
.id(manualRedactionEntry.getAnnotationId())
.value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType())
.entryType(manualRedactionEntry.getDictionaryEntryType().toEntryType())
.entryType(Optional.ofNullable(manualRedactionEntry.getDictionaryEntryType())
.orElse(DictionaryEntryType.ENTRY).toEntryType())
.state(EntryState.PENDING)
.dictionaryEntry(manualRedactionEntry.isAddToDictionary())
.dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -185,3 +185,5 @@ databaseChangeLog:
file: db/changelog/tenant/120-add-legal-basis-change-to-manual-recategorization.yaml
- include:
file: db/changelog/tenant/sql/205-add-dossier-dictionaries-as-entity.sql
- include:
file: db/changelog/tenant/121-set-dictionary-entry-type-for-dictionary-adds-where-null.yaml

13
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/121-set-dictionary-entry-type-for-dictionary-adds-where-null.yaml Normal file
View File

@ -0,0 +1,13 @@
databaseChangeLog:
- changeSet:
id: set-dictionary-entry-type-for-dictionary-adds-where-null
author: kilian
changes:
- update:
tableName: manual_redaction
columns:
- column:
name: dictionary_entry_type
value: 'ENTRY'
where: "add_to_dictionary AND dictionary_entry_type IS NULL"

288
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java
View File

@ -92,10 +92,15 @@ public class EntityLogMergeTest {
private EntityLogMergeService entityLogMergeService;
@BeforeEach
public void setUp() {
entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService, rabbitTemplate, fileStatusService, fileStatusPersistenceService, new PendingDictionaryEntryFactory());
entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService,
rabbitTemplate,
fileStatusService,
fileStatusPersistenceService,
new PendingDictionaryEntryFactory());
}
@ -118,27 +123,22 @@ public class EntityLogMergeTest {
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId);
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder()
.excluded(false)
.dossierStatusId(dossierTemplateId)
.id(fileId)
.build());
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
when(fileManagementStorageService.getEntityLog(dossierId, fileId)).thenReturn(entityLog);
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder()
.dossierTemplateId(dossierTemplateId)
.build());
when(dossierService.getDossierById(dossierId)).thenReturn(DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build());
when(fileStatusPersistenceService.getStatus(fileId)).thenReturn(FileEntity.builder().id(fileId).fileAttributes(Collections.emptyList()).build());
when(fileStatusService.convertAttributes(any(), anyString())).thenReturn(Collections.emptyList());
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder()
.dossierTemplateId(dossierTemplateId)
.build());
EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, entityLog, DossierEntity.builder().dossierTemplateId(dossierTemplateId).build());
assertNotNull(response);
assertFalse(response.getEntityLogEntry().isEmpty());
var optionalEntityLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToAddId)).findFirst();
var optionalEntityLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToAddId))
.findFirst();
assertTrue(optionalEntityLogEntry.isPresent());
var entityLogEntry = optionalEntityLogEntry.get();
assertEquals(entityLogEntry.getType(), "manual");
@ -150,54 +150,78 @@ public class EntityLogMergeTest {
assertEquals(entityLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.ADD);
var optionalRemoveEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRemoveId)).findFirst();
var optionalRemoveEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRemoveId))
.findFirst();
assertTrue(optionalRemoveEntryLogEntry.isPresent());
var removeEntryLogEntry = optionalRemoveEntryLogEntry.get();
assertEquals(removeEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(removeEntryLogEntry.getState(), EntryState.IGNORED);
assertEquals(removeEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
assertEquals(removeEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.REMOVE);
assertEquals(removeEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.REMOVED);
assertTrue(removeEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalResizeEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId)).findFirst();
var optionalResizeEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToResizeId))
.findFirst();
assertTrue(optionalResizeEntryLogEntry.isPresent());
var resizeEntryLogEntry = optionalResizeEntryLogEntry.get();
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[0], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[1], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[2], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getRectangle()[3], 2);
assertEquals(resizeEntryLogEntry.getPositions().get(0).getPageNumber(), 1);
assertEquals(resizeEntryLogEntry.getPositions()
.get(0).getRectangle()[0], 2);
assertEquals(resizeEntryLogEntry.getPositions()
.get(0).getRectangle()[1], 2);
assertEquals(resizeEntryLogEntry.getPositions()
.get(0).getRectangle()[2], 2);
assertEquals(resizeEntryLogEntry.getPositions()
.get(0).getRectangle()[3], 2);
assertEquals(resizeEntryLogEntry.getPositions()
.get(0).getPageNumber(), 1);
assertEquals(resizeEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(resizeEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(resizeEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertEquals(resizeEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertEquals(resizeEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.CHANGED);
assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId)).findFirst();
var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryLegalBasisId))
.findFirst();
assertTrue(optionalLegalBasisEntryLogEntry.isPresent());
var legalBasisEntryLogEntry = optionalLegalBasisEntryLogEntry.get();
assertEquals(legalBasisEntryLogEntry.getLegalBasis(), "New legal basis");
assertEquals(legalBasisEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(legalBasisEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(legalBasisEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.LEGAL_BASIS_CHANGE);
assertEquals(legalBasisEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.LEGAL_BASIS_CHANGE);
assertEquals(legalBasisEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.CHANGED);
assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId)).findFirst();
var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(forceRedactionId))
.findFirst();
assertTrue(optionalForceRedactionEntryLogEntry.isPresent());
var forceRedactionEntryLogEntry = optionalForceRedactionEntryLogEntry.get();
assertEquals(forceRedactionEntryLogEntry.getLegalBasis(), "Force");
assertEquals(forceRedactionEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(forceRedactionEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(forceRedactionEntryLogEntry.getManualChanges().get(0).getManualRedactionType(), ManualRedactionType.FORCE);
assertEquals(forceRedactionEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.FORCE);
assertEquals(forceRedactionEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.CHANGED);
assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalRectangleEntryLogEntry = response.getEntityLogEntry().stream().filter(entityLogEntry1 -> entityLogEntry1.getId().equals(rectangleToAddId)).findFirst();
var optionalRectangleEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(rectangleToAddId))
.findFirst();
assertTrue(optionalRectangleEntryLogEntry.isPresent());
var rectangleEntryLogEntry = optionalRectangleEntryLogEntry.get();
assertEquals(rectangleEntryLogEntry.getType(), "manual");
@ -211,119 +235,127 @@ public class EntityLogMergeTest {
}
private EntityLog provideEntityLog(String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId) {
List<Position> positions = new ArrayList<>();
positions.add(new Position(1, 1, 1, 1, 1));
return new EntityLog(1,
1,
Lists.newArrayList(EntityLogEntry.builder()
.id(entryToRemoveId)
.type("manual")
.value("Luke Skywalker")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.build(),
EntityLogEntry.builder()
.id(entryToResizeId)
.type("manual")
.value("Darth Vader")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(entryLegalBasisId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(forceRedactionId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build()),
null,
0,
0,
0,
0);
1,
Lists.newArrayList(EntityLogEntry.builder()
.id(entryToRemoveId)
.type("manual")
.value("Luke Skywalker")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.build(),
EntityLogEntry.builder()
.id(entryToResizeId)
.type("manual")
.value("Darth Vader")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(entryLegalBasisId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(forceRedactionId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(true)
.positions(positions)
.build()),
null,
0,
0,
0,
0);
}
private ManualRedactions provideManualRedactions(String entryToAddId, String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId, String fileId, String rectangleToAddId) {
private ManualRedactions provideManualRedactions(String entryToAddId,
String entryToRemoveId,
String entryToResizeId,
String entryLegalBasisId,
String forceRedactionId,
String fileId,
String rectangleToAddId) {
List<Rectangle> positions = new ArrayList<>();
positions.add(new Rectangle(2, 2, 2, 2, 1));
return ManualRedactions.builder()
.entriesToAdd(Set.of(
ManualRedactionEntry.builder()
.positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
.annotationId(entryToAddId)
.value("Test")
.reason("Reason")
.addToDictionary(false)
.addToDossierDictionary(false)
.fileId(fileId)
.rectangle(false)
.requestDate(OffsetDateTime.now())
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.type("manual")
.build(),
ManualRedactionEntry.builder()
.positions(List.of(new Rectangle(5f, 6f, 7f, 8f, 1)))
.annotationId(rectangleToAddId)
.value("Test2")
.reason("Rectangle")
.addToDictionary(false)
.addToDossierDictionary(false)
.fileId(fileId)
.rectangle(true)
.requestDate(OffsetDateTime.now())
.type("manual")
.build()))
.idsToRemove(Set.of(
IdRemoval.builder()
.annotationId(entryToRemoveId)
.requestDate(OffsetDateTime.now())
.build()
))
.resizeRedactions(Set.of(
ManualResizeRedaction.builder()
.fileId(fileId)
.value("Random")
.annotationId(entryToResizeId)
.positions(positions)
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.build()
))
.legalBasisChanges(Set.of(
ManualLegalBasisChange.builder()
.annotationId(entryLegalBasisId)
.value("Random")
.legalBasis("New legal basis")
.section("Section")
.requestDate(OffsetDateTime.now())
.build()
))
.forceRedactions(Set.of(
ManualForceRedaction.builder()
.annotationId(forceRedactionId)
.fileId(fileId)
.legalBasis("Force")
.requestDate(OffsetDateTime.now())
.build()
))
.entriesToAdd(Set.of(ManualRedactionEntry.builder()
.positions(List.of(new Rectangle(1f, 2f, 3f, 4f, 1)))
.annotationId(entryToAddId)
.value("Test")
.reason("Reason")
.addToDictionary(false)
.addToDossierDictionary(false)
.fileId(fileId)
.rectangle(false)
.fileId("file")
.requestDate(OffsetDateTime.now())
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.type("manual")
.user("User")
.build(),
ManualRedactionEntry.builder()
.positions(List.of(new Rectangle(5f, 6f, 7f, 8f, 1)))
.annotationId(rectangleToAddId)
.value("Test2")
.reason("Rectangle")
.addToDictionary(false)
.addToDossierDictionary(false)
.fileId(fileId)
.rectangle(true)
.requestDate(OffsetDateTime.now())
.type("manual")
.user("User")
.fileId("file")
.build()))
.idsToRemove(Set.of(IdRemoval.builder().annotationId(entryToRemoveId).requestDate(OffsetDateTime.now()).user("user").fileId("file").build()))
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.fileId(fileId)
.value("Random")
.annotationId(entryToResizeId)
.positions(positions)
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.user("User")
.fileId("file")
.build()))
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId(entryLegalBasisId)
.value("Random")
.legalBasis("New legal basis")
.user("User")
.section("Section")
.fileId("file")
.requestDate(OffsetDateTime.now())
.build()))
.forceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId(forceRedactionId)
.fileId(fileId)
.legalBasis("Force")
.requestDate(OffsetDateTime.now())
.user("User")
.fileId("file")
.build()))
.build();
}
}

5
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/BaseAnnotation.java
View File

@ -8,6 +8,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.experimental.SuperBuilder;
@Data
@ -17,9 +18,13 @@ import lombok.experimental.SuperBuilder;
@EqualsAndHashCode
public abstract class BaseAnnotation {
@NonNull
private String annotationId;
@NonNull
private String fileId;
@NonNull
private String user;
@NonNull
private OffsetDateTime requestDate;
private OffsetDateTime processedDate;
private OffsetDateTime softDeletedTime;