Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9140 - Add more information to changes #541

Merged
andrei.isvoran.ext merged 17 commits from RED-9140 into master 2024-06-27 11:53:36 +02:00
Conversation 21 Commits 17 Files Changed 21 +16445 -65
21 changed files with 16445 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/EntityLogController.java vendored
View File

@ -37,7 +37,10 @@ public class EntityLogController implements EntityLogResource {
accessControlService.checkViewPermissionsToDossier(dossierId);
accessControlService.validateFileResourceExistence(fileId);
return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLog(dossierId, fileId, excludedTypes == null ? new ArrayList<>() : excludedTypes, includeUnprocessed));
return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLog(dossierId,
fileId,
excludedTypes == null ? new ArrayList<>() : excludedTypes,
includeUnprocessed));
}
@ -64,13 +67,14 @@ public class EntityLogController implements EntityLogResource {
@PreAuthorize("hasAuthority('" + READ_REDACTION_LOG + "')")
public EntityLogResponse getEntityLogWithEntriesAnalysedAfterIteration(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@PathVariable(ANALYSIS_NUMBER) Integer analysisNumber) {
public EntityLogResponse getEntityLogDeltaUpdate(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@PathVariable(ANALYSIS_NUMBER) Integer analysisNumber,
@RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed) {
accessControlService.checkViewPermissionsToDossier(dossierId);
accessControlService.validateFileResourceExistence(fileId);
return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLogWithEntriesWithEntriesByAnalysisNumber(dossierId, fileId, analysisNumber));
return entityLogResponseMapper.toLogResponse(entityLogService.getEntityLogWithEntriesWithEntriesSinceAnalysisNumber(dossierId, fileId, analysisNumber, includeUnprocessed));
}
}

11
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/EntityLogResource.java vendored
View File

@ -55,14 +55,17 @@ public interface EntityLogResource {
@GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + "/pages", produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Gets the entity log for a fileId with all entities found on the given page numbers", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The dossier / file / entity log is not found.")})
EntityLogResponse getEntityLogWithEntriesOnPages(@PathVariable(DOSSIER_ID) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestParam(value = "pageNumbers", defaultValue = "") List<Integer> pageNumbers);
EntityLogResponse getEntityLogWithEntriesOnPages(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = "pageNumbers", defaultValue = "") List<Integer> pageNumbers);
@GetMapping(value = ENTITY_LOG_PATH + DOSSIER_ID_PATH_VARIABLE + FILE_ID_PATH_VARIABLE + ANALYSIS_NUMBER_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Gets the entity log for a fileId with all entities being analysed in or after iteration with given number", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The dossier / file / entity log is not found.")})
EntityLogResponse getEntityLogWithEntriesAnalysedAfterIteration(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@PathVariable(ANALYSIS_NUMBER) Integer analysisNumber);
EntityLogResponse getEntityLogDeltaUpdate(@PathVariable(DOSSIER_ID) String dossierId,
@PathVariable(FILE_ID) String fileId,
@PathVariable(ANALYSIS_NUMBER) Integer analysisNumber,
@RequestParam(value = "includeUnprocessed", required = false, defaultValue = FALSE) boolean includeUnprocessed);
}

136
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java
View File

@ -35,6 +35,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@ -46,6 +48,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.ChangeUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import io.micrometer.observation.annotation.Observed;
@ -279,10 +282,10 @@ public class EntityLogMergeService {
public EntityLogEntry buildEntityLogEntry(ManualRedactionEntry manualRedactionEntry, int analysisNumber, DossierEntity dossier) {
List<ManualChange> manualChanges = new ArrayList<>();
manualChanges.add(ManualChangeFactory.toLocalManualChange(manualRedactionEntry));
manualChanges.add(ManualChangeFactory.toLocalManualChange(manualRedactionEntry, 0));
List<Change> changes = new ArrayList<>();
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(manualRedactionEntry.getRequestDate()).type(ChangeType.ADDED).build());
changes.add(ChangeFactory.toChange(ChangeType.ADDED, manualRedactionEntry.getRequestDate(), analysisNumber));
boolean isHint = isHint(manualRedactionEntry.getType(), dossier);
@ -323,37 +326,66 @@ public class EntityLogMergeService {
}
private void mergeFalsePositive(int analysisNumber, EntityLogEntry existingEntry) {
private void mergeFalsePositive(int analysisNumber, EntityLogEntry entityLogEntry) {
existingEntry.setState(EntryState.REMOVED);
List<Change> falsePositiveChanges = new ArrayList<>();
falsePositiveChanges.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(ChangeType.REMOVED).build());
if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) {
existingEntry.getChanges().addAll(falsePositiveChanges);
} else {
existingEntry.setChanges(falsePositiveChanges);
}
List<Change> changes = new ArrayList<>();
changes.add(ChangeFactory.toChange(ChangeType.REMOVED,
OffsetDateTime.now(),
analysisNumber,
PropertyChange.builder().property("state").oldValue(entityLogEntry.getState().name()).newValue(EntryState.REMOVED.name()).build()));
addChanges(entityLogEntry, changes);
entityLogEntry.setState(EntryState.REMOVED);
}
public void mergeIdToRemove(IdRemoval idRemoval, EntityLogEntry entityLogEntry, int analysisNumber) {
List<Change> changes = new ArrayList<>();
changes.add(ChangeFactory.toChange(ChangeType.REMOVED,
OffsetDateTime.now(),
analysisNumber,
PropertyChange.builder().property("state").oldValue(entityLogEntry.getState().name()).newValue(EntryState.IGNORED.name()).build()));
addChanges(entityLogEntry, changes);
entityLogEntry.setState(EntryState.IGNORED);
entityLogEntry.getEngines().add(Engine.MANUAL);
addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval));
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(idRemoval, 0));
}
public void mergeResizeRedaction(ManualResizeRedaction manualResizeRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
List<Change> changes = new ArrayList<>();
List<Position> currentPositions = entityLogEntry.getPositions();
List<Position> newPositions = convertPositions(manualResizeRedaction.getPositions());
changes.add(ChangeFactory.toChange(ChangeType.RESIZED,
manualResizeRedaction.getRequestDate(),
analysisNumber,
PropertyChange.builder()
.property("textAfter")
.oldValue(entityLogEntry.getTextAfter())
.newValue(manualResizeRedaction.getTextAfter())
.build(),
PropertyChange.builder()
.property("textBefore")
.oldValue(entityLogEntry.getTextBefore())
.newValue(manualResizeRedaction.getTextBefore())
.build(),
PropertyChange.builder().property("value").oldValue(entityLogEntry.getValue()).newValue(manualResizeRedaction.getValue()).build(),
PropertyChange.builder()
.property("positions")
.oldValue(ChangeUtils.prettifyPositions(currentPositions))
.newValue(ChangeUtils.prettifyPositions(newPositions))
.build()));
addChanges(entityLogEntry, changes);
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
entityLogEntry.setPositions(newPositions);
entityLogEntry.setValue(manualResizeRedaction.getValue());
entityLogEntry.getEngines().add(Engine.MANUAL);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualResizeRedaction.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualResizeRedaction));
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualResizeRedaction, 0));
}
@ -365,21 +397,41 @@ public class EntityLogMergeService {
public void mergeLegalBasisChange(ManualLegalBasisChange manualLegalBasisChange, EntityLogEntry entityLogEntry, int analysisNumber) {
List<Change> changes = new ArrayList<>();
changes.add(ChangeFactory.toChange(ChangeType.LEGAL_BASIS_CHANGE,
manualLegalBasisChange.getRequestDate(),
analysisNumber,
PropertyChange.builder()
.property("legalBasis")
.oldValue(entityLogEntry.getLegalBasis())
.newValue(manualLegalBasisChange.getLegalBasis())
.build(),
PropertyChange.builder().property("section").oldValue(entityLogEntry.getSection()).newValue(manualLegalBasisChange.getSection()).build(),
PropertyChange.builder().property("value").oldValue(entityLogEntry.getValue()).newValue(manualLegalBasisChange.getValue()).build()));
addChanges(entityLogEntry, changes);
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
entityLogEntry.getEngines().add(Engine.MANUAL);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualLegalBasisChange));
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(manualLegalBasisChange, 0));
}
public EntityLogEntry mergeRecategorization(ManualRecategorization recategorization, EntityLogEntry entityLogEntry, DossierEntity dossier, int analysisNumber) {
List<Change> changes = new ArrayList<>();
Change change = ChangeFactory.toChange(ChangeType.RECATEGORIZE,
recategorization.getRequestDate(),
analysisNumber,
PropertyChange.builder().property("type").oldValue(entityLogEntry.getType()).newValue(recategorization.getType()).build());
changes.add(change);
EntryState entryState = entityLogEntry.getState();
if ((recategorization.getType() != null && !Objects.equals(recategorization.getType(), entityLogEntry.getType()) && Strings.isNullOrEmpty(recategorization.getLegalBasis()))
//
&& (entityLogEntry.getEntryType().equals(EntryType.IMAGE) || entityLogEntry.getEntryType().equals(EntryType.IMAGE_HINT))) {
addChanges(entityLogEntry, changes);
return pendingDictionaryEntryFactory.buildPendingImageRecategorizationEntry(recategorization, entityLogEntry);
}
@ -388,37 +440,63 @@ public class EntityLogMergeService {
if (recategorization.getType() != null && !recategorization.getType().equals(entityLogEntry.getType())) {
boolean isHint = isHint(recategorization.getType(), dossier);
entityLogEntry.setType(recategorization.getType());
entityLogEntry.setEntryType(getEntryType(isHint, recategorization.getType()));
EntryType entryType = getEntryType(isHint, recategorization.getType());
ChangeFactory.addPropertyChanges(change,
PropertyChange.builder().property("entryType").oldValue(entityLogEntry.getEntryType().name()).newValue(entryType.name()).build());
entityLogEntry.setEntryType(entryType);
entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
}
if (!Strings.isNullOrEmpty(recategorization.getLegalBasis())) {
boolean isHint = isHint(entityLogEntry.getType(), dossier);
ChangeFactory.addPropertyChanges(change,
PropertyChange.builder()
.property("legalBasis")
.oldValue(entityLogEntry.getLegalBasis())
.newValue(recategorization.getLegalBasis())
.build());
entityLogEntry.setLegalBasis(recategorization.getLegalBasis());
entityLogEntry.setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED);
}
ChangeFactory.addPropertyChanges(change, PropertyChange.builder().property("state").oldValue(entryState.name()).newValue(entityLogEntry.getState().name()).build());
if (recategorization.getSection() != null) {
ChangeFactory.addPropertyChanges(change,
PropertyChange.builder().property("section").oldValue(entityLogEntry.getSection()).newValue(recategorization.getSection()).build());
entityLogEntry.setSection(recategorization.getSection());
}
if (recategorization.getValue() != null) {
ChangeFactory.addPropertyChanges(change, PropertyChange.builder().property("value").oldValue(entityLogEntry.getValue()).newValue(recategorization.getValue()).build());
entityLogEntry.setValue(recategorization.getValue());
}
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, recategorization.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(recategorization));
addChanges(entityLogEntry, changes);
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(recategorization, 0));
return null;
}
public void mergeForceRedaction(ManualForceRedaction forceRedaction, EntityLogEntry entityLogEntry, int analysisNumber) {
List<Change> changes = new ArrayList<>();
EntryState oldState = entityLogEntry.getState();
EntryState newState = entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED;
changes.add(ChangeFactory.toChange(ChangeType.FORCE_REDACT,
forceRedaction.getRequestDate(),
analysisNumber,
PropertyChange.builder()
.property("legalBasis")
.oldValue(entityLogEntry.getLegalBasis())
.newValue(forceRedaction.getLegalBasis())
.build(),
PropertyChange.builder().property("state").oldValue(oldState.name()).newValue(newState.name()).build()));
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED);
entityLogEntry.setState(newState);
entityLogEntry.getEngines().add(Engine.MANUAL);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(forceRedaction));
addChanges(entityLogEntry, changes);
entityLogEntry.getManualChanges().add(ManualChangeFactory.toLocalManualChange(forceRedaction, 0));
}
@ -432,12 +510,12 @@ public class EntityLogMergeService {
}
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
private void addChanges(EntityLogEntry entityLogEntry, List<Change> changes) {
if (!changes.isEmpty()) {
changes.add(Change.builder().analysisNumber(analysisNumber + 1).dateTime(offsetDateTime).type(changeType).build());
if (entityLogEntry.getChanges() != null && !entityLogEntry.getChanges().isEmpty()) {
entityLogEntry.getChanges().addAll(changes);
} else {
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
entityLogEntry.setChanges(changes);
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java
View File

@ -142,10 +142,10 @@ public class EntityLogService {
@Observed(name = "EntityLogService", contextualName = "get-entity-log")
public EntityLog getEntityLogWithEntriesWithEntriesByAnalysisNumber(String dossierId, String fileId, Integer analysisNumber) {
public EntityLog getEntityLogWithEntriesWithEntriesSinceAnalysisNumber(String dossierId, String fileId, Integer analysisNumber, boolean includeUnprocessed) {
EntityLog entityLog = fileManagementStorageService.getEntityLogWithEntriesByAnalysisNumber(dossierId, fileId, analysisNumber);
postProcessEntityLog(dossierId, fileId, entityLog, false);
EntityLog entityLog = fileManagementStorageService.getEntityLogWithEntriesSinceAnalysisNumber(dossierId, fileId, analysisNumber);
postProcessEntityLog(dossierId, fileId, entityLog, includeUnprocessed);
return entityLog;
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -117,9 +117,9 @@ public class FileManagementStorageService {
}
public EntityLog getEntityLogWithEntriesByAnalysisNumber(String dossierId, String fileId, Integer analysisNumber) {
public EntityLog getEntityLogWithEntriesSinceAnalysisNumber(String dossierId, String fileId, Integer analysisNumber) {
return entityLogMongoService.findEntityLogWithEntriesByAnalysisNumber(dossierId, fileId, analysisNumber)
return entityLogMongoService.findEntityLogWithEntriesSinceAnalysisNumber(dossierId, fileId, analysisNumber)
.orElseThrow(() -> new NotFoundException(getEntityLogNotFoundErrorMessage(dossierId, fileId)));
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/manualredactions/PendingDictionaryEntryFactory.java
View File

@ -208,7 +208,7 @@ public class PendingDictionaryEntryFactory {
public EntityLogEntry buildPendingImageRecategorizationEntry(ManualRecategorization manualChange, EntityLogEntry originalEntry) {
var manualChanges = List.of(ManualChangeFactory.toLocalManualChange(manualChange));
var manualChanges = List.of(ManualChangeFactory.toLocalManualChange(manualChange, 0));
String reason = String.format("Image has been recategorized from %s to %s", originalEntry.getType(), manualChange.getType());

65
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java
View File

@ -52,6 +52,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
@ -117,10 +118,11 @@ public class EntityLogMergeTest {
String entryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
String entryToRecategorizeId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId);
ManualRedactions manualRedactions = provideManualRedactions(entryToAddId, entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, fileId, rectangleToAddId, entryToRecategorizeId);
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, false);
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, false);
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
@ -185,7 +187,9 @@ public class EntityLogMergeTest {
assertEquals(resizeEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RESIZE);
assertEquals(resizeEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.CHANGED);
.get(0).getType(), ChangeType.RESIZED);
assertEquals(resizeEntryLogEntry.getChanges().get(0).getPropertyChanges().get("value"), "Darth Vader -> Random");
assertEquals(resizeEntryLogEntry.getChanges().get(0).getPropertyChanges().get("positions"), "[1.0, 1.0, 1.0, 1.0, 1] -> [2.0, 2.0, 2.0, 2.0, 1]");
assertTrue(resizeEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalLegalBasisEntryLogEntry = response.getEntityLogEntry()
@ -200,7 +204,8 @@ public class EntityLogMergeTest {
assertEquals(legalBasisEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.LEGAL_BASIS_CHANGE);
assertEquals(legalBasisEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.CHANGED);
.get(0).getType(), ChangeType.LEGAL_BASIS_CHANGE);
assertEquals(legalBasisEntryLogEntry.getChanges().get(0).getPropertyChanges().get("legalBasis"), "legalBasis -> New legal basis");
assertTrue(legalBasisEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalForceRedactionEntryLogEntry = response.getEntityLogEntry()
@ -215,7 +220,9 @@ public class EntityLogMergeTest {
assertEquals(forceRedactionEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.FORCE);
assertEquals(forceRedactionEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.CHANGED);
.get(0).getType(), ChangeType.FORCE_REDACT);
assertEquals(forceRedactionEntryLogEntry.getChanges().get(0).getPropertyChanges().get("legalBasis"), "null -> Force");
assertEquals(forceRedactionEntryLogEntry.getChanges().get(0).getPropertyChanges().get("state"), "SKIPPED -> APPLIED");
assertTrue(forceRedactionEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalRectangleEntryLogEntry = response.getEntityLogEntry()
@ -233,6 +240,22 @@ public class EntityLogMergeTest {
.get(0).getManualRedactionType(), ManualRedactionType.ADD);
assertTrue(rectangleEntryLogEntry.getEngines().contains(Engine.MANUAL));
var optionalRecategorizationEntryLogEntry = response.getEntityLogEntry()
.stream()
.filter(entityLogEntry1 -> entityLogEntry1.getId().equals(entryToRecategorizeId))
.findFirst();
var recategorizationEntryLogEntry = optionalRecategorizationEntryLogEntry.get();
assertEquals(recategorizationEntryLogEntry.getLegalBasis(), "New legal basis");
assertEquals(recategorizationEntryLogEntry.getEntryType(), EntryType.ENTITY);
assertEquals(recategorizationEntryLogEntry.getType(), "PII");
assertEquals(recategorizationEntryLogEntry.getState(), EntryState.APPLIED);
assertEquals(recategorizationEntryLogEntry.getManualChanges()
.get(0).getManualRedactionType(), ManualRedactionType.RECATEGORIZE);
assertEquals(recategorizationEntryLogEntry.getChanges()
.get(0).getType(), ChangeType.RECATEGORIZE);
assertEquals(recategorizationEntryLogEntry.getChanges().get(0).getPropertyChanges().get("legalBasis"), "legalBasis -> New legal basis");
assertEquals(recategorizationEntryLogEntry.getChanges().get(0).getPropertyChanges().get("type"), "CBI_author -> PII");
assertTrue(recategorizationEntryLogEntry.getEngines().contains(Engine.MANUAL));
}
@ -248,6 +271,7 @@ public class EntityLogMergeTest {
String entryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
String entryToRecategorizeId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = provideManualRedactions(entryLegalBasisId,
entryToRemoveId,
@ -255,9 +279,9 @@ public class EntityLogMergeTest {
entryLegalBasisId,
forceRedactionId,
fileId,
rectangleToAddId);
rectangleToAddId, entryToRecategorizeId);
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, true);
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, true);
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
@ -307,6 +331,7 @@ public class EntityLogMergeTest {
String entryToResizeId = UUID.randomUUID().toString();
String entryLegalBasisId = UUID.randomUUID().toString();
String forceRedactionId = UUID.randomUUID().toString();
String entryToRecategorizeId = UUID.randomUUID().toString();
ManualRedactions manualRedactions = new ManualRedactions();
List<Rectangle> positions = new ArrayList<>();
@ -322,7 +347,7 @@ public class EntityLogMergeTest {
.fileId("file")
.build()));
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, false);
var entityLog = provideEntityLog(entryToRemoveId, entryToResizeId, entryLegalBasisId, forceRedactionId, entryToRecategorizeId, false);
when(manualRedactionProviderService.getManualRedactions(any(), any())).thenReturn(manualRedactions);
when(fileStatusService.getStatus(fileId)).thenReturn(FileModel.builder().excluded(false).dossierStatusId(dossierTemplateId).id(fileId).build());
@ -347,7 +372,7 @@ public class EntityLogMergeTest {
}
private EntityLog provideEntityLog(String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId, boolean dictEntry) {
private EntityLog provideEntityLog(String entryToRemoveId, String entryToResizeId, String entryLegalBasisId, String forceRedactionId, String entryToRecategorizeId, boolean dictEntry) {
List<Position> positions = new ArrayList<>();
positions.add(new Position(1, 1, 1, 1, 1));
@ -378,15 +403,27 @@ public class EntityLogMergeTest {
.state(EntryState.APPLIED)
.dictionaryEntry(dictEntry)
.positions(positions)
.legalBasis("legalBasis")
.build(),
EntityLogEntry.builder()
.id(forceRedactionId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.SKIPPED)
.dictionaryEntry(dictEntry)
.positions(positions)
.build(),
EntityLogEntry.builder()
.id(entryToRecategorizeId)
.type("manual")
.value("Darth Luke")
.entryType(EntryType.ENTITY)
.state(EntryState.APPLIED)
.dictionaryEntry(dictEntry)
.positions(positions)
.legalBasis("legalBasis")
.type("CBI_author")
.build()),
Collections.emptyList(),
0,
@ -402,7 +439,7 @@ public class EntityLogMergeTest {
String entryLegalBasisId,
String forceRedactionId,
String fileId,
String rectangleToAddId) {
String rectangleToAddId, String entryToRecategorizeId) {
List<Rectangle> positions = new ArrayList<>();
positions.add(new Rectangle(2, 2, 2, 2, 1));
@ -465,6 +502,14 @@ public class EntityLogMergeTest {
.user("User")
.fileId("file")
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId(entryToRecategorizeId)
.fileId(fileId)
.legalBasis("New legal basis")
.requestDate(OffsetDateTime.now())
.user("User")
.type("PII")
.build()))
.build();
}

21
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMongoServiceTest.java
View File

@ -29,6 +29,7 @@ public class EntityLogMongoServiceTest extends AbstractPersistenceServerServiceT
private final String ENTITY_LOG2 = "files/entity-log/6f2a9d4b5d4e11211fc0d7732fd45b78.ENTITY_LOG.json";
private final String ENTITY_LOG3_BEFORE = "files/entity-log/c3b23116f2d277170d303bfc6fb82e6a-before.ENTITY_LOG.json";
private final String ENTITY_LOG3_AFTER = "files/entity-log/c3b23116f2d277170d303bfc6fb82e6a-after.ENTITY_LOG.json";
private final String ENTITY_LOG_WITH_CHANGES = "files/entity-log/entitylog-with-changes.json";
private static final String TEST_DOSSIER_ID = "91ce8e90-9aec-473c-b8c3-cbe16443ad34";
private static final String TEST_FILE1_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
@ -305,4 +306,24 @@ public class EntityLogMongoServiceTest extends AbstractPersistenceServerServiceT
}
@Test
@SneakyThrows
public void testGetEntityLogDeltaUpdate() {
var file = new ClassPathResource(ENTITY_LOG_WITH_CHANGES);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_ID, TEST_FILE1_ID, entityLog);
Optional<EntityLog> optionalEntityLog = entityLogMongoService.findEntityLogWithEntriesSinceAnalysisNumber(TEST_DOSSIER_ID, TEST_FILE1_ID, 2);
assertTrue(optionalEntityLog.isPresent());
EntityLog response = optionalEntityLog.get();
assertEquals(response.getEntityLogEntry().size(), 14);
}
}

5
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -15,6 +15,7 @@ import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@ -2378,7 +2379,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.dictionaryEntry(false)
.dossierDictionaryEntry(false)
.excluded(false)
.changes(List.of(new Change(1, ChangeType.ADDED, OffsetDateTime.now())))
.changes(List.of(new Change(1, ChangeType.ADDED, OffsetDateTime.now(), Collections.emptyMap())))
.build();
var entityLog = new EntityLog(1,
1,
@ -2447,7 +2448,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.dictionaryEntry(true)
.dossierDictionaryEntry(false)
.excluded(false)
.changes(List.of(new Change(1, ChangeType.ADDED, OffsetDateTime.now())))
.changes(List.of(new Change(1, ChangeType.ADDED, OffsetDateTime.now(), Collections.emptyMap())))
.engines(Set.of(Engine.DICTIONARY))
.build();
var entityLog = new EntityLog(1,

16093
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/entitylog-with-changes.json Normal file
View File

File diff suppressed because it is too large Load Diff

13
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/Change.java
View File

@ -1,6 +1,8 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
@ -16,5 +18,16 @@ public class Change {
private int analysisNumber;
private ChangeType type;
private OffsetDateTime dateTime;
private Map<String, String> propertyChanges = new HashMap<>();
public void addChange(String property, String value) {
if (value == null) {
return;
}
this.propertyChanges.put(property, value);
}
}

7
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/ChangeType.java
View File

@ -3,5 +3,10 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislo
public enum ChangeType {
ADDED,
REMOVED,
CHANGED
CHANGED,
RESIZED,
LEGAL_BASIS_CHANGE,
RECATEGORIZE,
FORCE_REDACT,
SYSTEM
}

4
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/ManualChange.java
View File

@ -22,14 +22,16 @@ public class ManualChange {
private OffsetDateTime requestedDate;
private String userId;
private Map<String, String> propertyChanges = new HashMap<>();
private int processedAnalysisNumber;
public static ManualChange from(BaseAnnotation baseAnnotation) {
public static ManualChange from(BaseAnnotation baseAnnotation, int processedAnalysisNumber) {
ManualChange manualChange = new ManualChange();
manualChange.processedDate = baseAnnotation.getProcessedDate();
manualChange.requestedDate = baseAnnotation.getRequestDate();
manualChange.userId = baseAnnotation.getUser();
manualChange.processedAnalysisNumber = processedAnalysisNumber;
return manualChange;
}

7
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/Position.java
View File

@ -61,4 +61,11 @@ public class Position {
return rectangle[3];
}
@Override
public String toString() {
return "[" + x() + ", " + y() + ", " + w() + ", " + h() + ", " + pageNumber + "]";
}
}

17
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/PropertyChange.java Normal file
View File

@ -0,0 +1,17 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PropertyChange {
private String property;
private String oldValue;
private String newValue;
}

44
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ChangeFactory.java Normal file
View File

@ -0,0 +1,44 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.annotations;
import java.time.OffsetDateTime;
import java.util.Arrays;
import java.util.Objects;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.ChangeUtils;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@UtilityClass
@Slf4j
public class ChangeFactory {
public Change toChange(ChangeType changeType, OffsetDateTime offsetDateTime, int analysisNumber, PropertyChange... propertyChanges) {
Change change = new Change();
change.setDateTime(offsetDateTime);
change.setAnalysisNumber(analysisNumber);
change.setType(changeType);
addPropertyChanges(change, propertyChanges);
return change;
}
public void addPropertyChanges(Change change, PropertyChange... propertyChanges) {
Arrays.stream(propertyChanges)
.forEach(propertyChange -> {
if (!Objects.equals(propertyChange.getOldValue(), propertyChange.getNewValue())
&& !propertyChange.getProperty().equals("manualChanges")
&& !propertyChange.getProperty().equals("changes")) {
change.addChange(propertyChange.getProperty(), ChangeUtils.buildVisualChange(propertyChange.getOldValue(), propertyChange.getNewValue()));
}
});
}
}

8
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/ManualChangeFactory.java
View File

@ -20,10 +20,10 @@ import lombok.experimental.UtilityClass;
@UtilityClass
public class ManualChangeFactory {
public List<ManualChange> toLocalManualChangeList(List<BaseAnnotation> manualChanges, boolean markAsProcessed) {
public List<ManualChange> toLocalManualChangeList(List<BaseAnnotation> manualChanges, boolean markAsProcessed, int processedAnalysisNumber) {
return manualChanges.stream()
.map(ManualChangeFactory::toLocalManualChange)
.map(manualChange -> toLocalManualChange(manualChange, processedAnalysisNumber))
.peek(manualChange -> {
if (markAsProcessed) {
manualChange.setProcessedDate(OffsetDateTime.now());
@ -33,13 +33,13 @@ public class ManualChangeFactory {
}
public ManualChange toLocalManualChange(BaseAnnotation baseAnnotation) {
public ManualChange toLocalManualChange(BaseAnnotation baseAnnotation, int processedAnalysisNumber) {
if (!baseAnnotation.isLocal()) {
throw new IllegalArgumentException(String.format("Manual change %s is not a local manual change", baseAnnotation));
}
ManualChange manualChange = ManualChange.from(baseAnnotation);
ManualChange manualChange = ManualChange.from(baseAnnotation, processedAnalysisNumber);
if (baseAnnotation instanceof ManualRecategorization recategorization) {
manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
.withChange("type", recategorization.getType())

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/Rectangle.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.annotations;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import lombok.AllArgsConstructor;

44
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/utils/ChangeUtils.java Normal file
View File

@ -0,0 +1,44 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.utils;
import java.util.List;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import lombok.experimental.UtilityClass;
@UtilityClass
public class ChangeUtils {
public String buildVisualChange(String oldValue, String newValue) {
String result = oldValue + " -> " + newValue;
String shortenedOldValue = oldValue;
String shortenedNewValue = oldValue;
if (result.length() > 200) {
if (oldValue.length() > 100) {
shortenedOldValue = oldValue.substring(0, 100) + "...";
}
if (newValue.length() > 100) {
shortenedNewValue = newValue.substring(0, 100) + "...";
}
result = shortenedOldValue + " -> " + shortenedNewValue;
}
return result;
}
public String prettifyPositions(List<Position> positions) {
if (positions.size() == 1) {
return positions.get(0).toString();
} else {
return positions.stream()
.map(Position::toString)
.collect(Collectors.joining(", ", "[", "]"));
}
}
}

4
persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java
View File

@ -21,6 +21,10 @@ public interface EntityLogEntryDocumentRepository extends MongoRepository<Entity
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumberEquals(String entityLogId, Integer analysisNumber);
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : { $gte: ?1, $lte: ?2 } }")
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumberBetween(String entityLogId, Integer analysisNumberStart, Integer analysisNumberEnd);
@Query("{ 'entityLogId' : ?0}")
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);

4
persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java
View File

@ -314,13 +314,13 @@ public class EntityLogMongoService {
}
public Optional<EntityLog> findEntityLogWithEntriesByAnalysisNumber(String dossierId, String fileId, Integer analysisNumber) {
public Optional<EntityLog> findEntityLogWithEntriesSinceAnalysisNumber(String dossierId, String fileId, Integer analysisNumber) {
String entityLogId = mapper.getLogId(dossierId, fileId);
Optional<EntityLog> optionalEntityLog = entityLogDocumentRepository.findEntityLogDocumentWithoutEntriesById(entityLogId)
.map(mapper::fromLogDocument);
optionalEntityLog.ifPresent(entityLog -> entityLog.getEntityLogEntry()
.addAll(entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumberEquals(entityLogId, analysisNumber)
.addAll(entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumberBetween(entityLogId, analysisNumber, entityLog.getAnalysisNumber())
.stream()
.map(mapper::fromLogEntryDocument)
.toList()));