RED-1908: Removed RedactionChangeLog added changes to RedactionLog
This commit is contained in:
Dominique Eifländer
parent
ff7166ed96
commit
e8a4ef172c
@ -0,0 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Change {
|
||||
|
||||
private ChangeType type;
|
||||
private OffsetDateTime dateTime;
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ChangeType {
|
||||
ADDED, REMOVED
|
||||
ADDED, REMOVED, CHANGED
|
||||
}
|
||||
|
||||
@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionChangeLog {
|
||||
|
||||
private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>();
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
private long dossierDictionaryVersion = -1;
|
||||
private long rulesVersion = -1;
|
||||
private long legalBasisVersion = -1;
|
||||
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionChangeLogEntry {
|
||||
|
||||
private String id;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private int matchedRule;
|
||||
private String legalBasis;
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
private String section;
|
||||
private float[] color;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private int sectionNumber;
|
||||
private boolean manual;
|
||||
private Status status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
@Builder.Default
|
||||
private List<Comment> comments = new ArrayList<>();
|
||||
|
||||
private ChangeType changeType;
|
||||
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
private boolean excluded;
|
||||
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLogChanges {
|
||||
|
||||
private RedactionLog redactionLog;
|
||||
private boolean hasChanges;
|
||||
|
||||
}
|
||||
@ -58,4 +58,8 @@ public class RedactionLogEntry {
|
||||
private String recategorizationType;
|
||||
private String legalBasisChangeValue;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
@Builder.Default
|
||||
private List<Change> changes = new ArrayList<>();
|
||||
|
||||
}
|
||||
|
||||
@ -1,12 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@ -17,7 +15,7 @@ public class AnalyzeResponseService {
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount,
|
||||
RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) {
|
||||
RedactionLog redactionLog, boolean hasUpdates) {
|
||||
|
||||
boolean hasHints = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
@ -41,12 +39,6 @@ public class AnalyzeResponseService {
|
||||
.filter(entry -> !entry.isExcluded())
|
||||
.anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage());
|
||||
|
||||
boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog
|
||||
.getRedactionLogEntry()
|
||||
.isEmpty() && redactionChangeLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.anyMatch(entry -> !entry.getType().equals("false_positive"));
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
|
||||
@ -78,9 +78,9 @@ public class ReanalyzeService {
|
||||
log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc
|
||||
.getDictionaryVersion(), analyzeRequest.getDossierTemplateId());
|
||||
|
||||
// first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data
|
||||
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
// store redactionLog
|
||||
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
redactionLog = redactionLogChange.getRedactionLog();
|
||||
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc
|
||||
.getSectionText()));
|
||||
@ -88,7 +88,7 @@ public class ReanalyzeService {
|
||||
.getSectionGrid());
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog);
|
||||
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, redactionLogChange.isHasChanges());
|
||||
}
|
||||
|
||||
|
||||
@ -266,13 +266,13 @@ public class ReanalyzeService {
|
||||
|
||||
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
|
||||
|
||||
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
|
||||
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
|
||||
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text
|
||||
.getNumberOfPages(), redactionLog, changeLog);
|
||||
.getNumberOfPages(), redactionLogChange.getRedactionLog(), redactionLogChange.isHasChanges());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,19 +1,25 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.file.management.v1.api.model.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.ChangeType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import com.iqser.red.service.redaction.v1.model.Change;
|
||||
import com.iqser.red.service.redaction.v1.model.ChangeType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogChanges;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -22,76 +28,77 @@ public class RedactionChangeLogService {
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) {
|
||||
|
||||
try {
|
||||
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
|
||||
var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog);
|
||||
redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog);
|
||||
return changeLog;
|
||||
} catch (Exception e) {
|
||||
log.debug("Previous redaction log not available");
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) {
|
||||
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) {
|
||||
|
||||
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
|
||||
|
||||
if (previousRedactionLog == null) {
|
||||
return null;
|
||||
currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
|
||||
entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
|
||||
});
|
||||
return new RedactionLogChanges(currentRedactionLog, false);
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry());
|
||||
added.removeAll(previousRedactionLog.getRedactionLogEntry());
|
||||
List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> !entry.getChanges()
|
||||
.get(entry.getChanges().size() - 1)
|
||||
.getType()
|
||||
.equals(ChangeType.REMOVED))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<RedactionLogEntry> removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry());
|
||||
Set<RedactionLogEntry> added = new HashSet<>(currentRedactionLog.getRedactionLogEntry());
|
||||
added.removeAll(notRemovedPreviousEntries);
|
||||
|
||||
Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
|
||||
removed.removeAll(currentRedactionLog.getRedactionLogEntry());
|
||||
|
||||
List<RedactionChangeLogEntry> changeLogEntries = added.stream()
|
||||
.map(entry -> convert(entry, ChangeType.ADDED))
|
||||
.collect(Collectors.toList());
|
||||
changeLogEntries.addAll(removed.stream()
|
||||
.map(entry -> convert(entry, ChangeType.REMOVED))
|
||||
.collect(Collectors.toList()));
|
||||
Map<String, RedactionLogEntry> addedIds = new HashMap<>();
|
||||
added.forEach(entry -> {
|
||||
addedIds.put(entry.getId(), entry);
|
||||
});
|
||||
|
||||
return new RedactionChangeLog(changeLogEntries,
|
||||
currentRedactionLog.getDictionaryVersion(),
|
||||
currentRedactionLog.getDossierDictionaryVersion(),
|
||||
currentRedactionLog.getRulesVersion(),
|
||||
currentRedactionLog.getLegalBasisVersion());
|
||||
}
|
||||
Set<String> removedIds = new HashSet<>();
|
||||
removed.forEach(entry -> {
|
||||
removedIds.add(entry.getId());
|
||||
});
|
||||
|
||||
List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
|
||||
|
||||
private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) {
|
||||
List<RedactionLogEntry> toRemove = new ArrayList<>();
|
||||
newRedactionLogEntries.forEach(entry -> {
|
||||
if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
|
||||
List<Change> changes = entry.getChanges();
|
||||
changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now()));
|
||||
var newEntry = addedIds.get(entry.getId());
|
||||
newEntry.setChanges(changes);
|
||||
addedIds.put(entry.getId(), newEntry);
|
||||
toRemove.add(entry);
|
||||
} else if (removedIds.contains(entry.getId())) {
|
||||
entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now()));
|
||||
} else if (addedIds.containsKey(entry.getId())) {
|
||||
List<Change> changes = entry.getChanges();
|
||||
changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
|
||||
var newEntry = addedIds.get(entry.getId());
|
||||
newEntry.setChanges(changes);
|
||||
addedIds.put(entry.getId(), newEntry);
|
||||
toRemove.add(entry);
|
||||
}
|
||||
});
|
||||
|
||||
return RedactionChangeLogEntry.builder()
|
||||
.id(entry.getId())
|
||||
.type(entry.getType())
|
||||
.value(entry.getValue())
|
||||
.reason(entry.getReason())
|
||||
.matchedRule(entry.getMatchedRule())
|
||||
.legalBasis(entry.getLegalBasis())
|
||||
.redacted(entry.isRedacted())
|
||||
.isHint(entry.isHint())
|
||||
.isRecommendation(entry.isRecommendation())
|
||||
.section(entry.getSection())
|
||||
.color(entry.getColor())
|
||||
.positions(entry.getPositions())
|
||||
.sectionNumber(entry.getSectionNumber())
|
||||
.manual(entry.isManual())
|
||||
.status(entry.getStatus())
|
||||
.manualRedactionType(entry.getManualRedactionType())
|
||||
.isDictionaryEntry(entry.isDictionaryEntry())
|
||||
.textBefore(entry.getTextBefore())
|
||||
.textAfter(entry.getTextAfter())
|
||||
.comments(entry.getComments())
|
||||
.changeType(changeType)
|
||||
.isDossierDictionaryEntry(entry.isDossierDictionaryEntry())
|
||||
.excluded(entry.isExcluded())
|
||||
.build();
|
||||
newRedactionLogEntries.removeAll(toRemove);
|
||||
|
||||
addedIds.forEach((k, v) -> {
|
||||
if(v.getChanges().isEmpty()) {
|
||||
v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
|
||||
}
|
||||
newRedactionLogEntries.add(v);
|
||||
});
|
||||
|
||||
currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
|
||||
|
||||
return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -714,10 +714,13 @@ public class RedactionIntegrationTest {
|
||||
reanlysisVersions.put("physical", 2L);
|
||||
|
||||
deleted.add("David Chubb");
|
||||
deleted.add("mouse");
|
||||
|
||||
dictionary.get(FALSE_POSITIVE).add("David Chubb");
|
||||
reanlysisVersions.put("David Chubb", 3L);
|
||||
|
||||
reanlysisVersions.put("mouse", 3L);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
|
||||
@ -754,6 +757,20 @@ public class RedactionIntegrationTest {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
|
||||
deleted.remove("mouse");
|
||||
reanlysisVersions.put("mouse", 4L);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L);
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
|
||||
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
|
||||
|
||||
reanalyzeService.reanalyze(request);
|
||||
|
||||
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
System.out.println("hi");
|
||||
}
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user