From e8a4ef172c994ab8dd99f84f7a725e3dddfaca5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Wed, 11 Aug 2021 09:45:00 +0200 Subject: [PATCH] RED-1908: Removed RedactionChangeLog added changes to RedactionLog --- .../service/redaction/v1/model/Change.java | 18 +++ .../redaction/v1/model/ChangeType.java | 2 +- .../v1/model/RedactionChangeLog.java | 22 --- .../v1/model/RedactionChangeLogEntry.java | 49 ------ .../v1/model/RedactionLogChanges.java | 17 ++ .../redaction/v1/model/RedactionLogEntry.java | 4 + .../service/AnalyzeResponseService.java | 12 +- .../redaction/service/ReanalyzeService.java | 14 +- .../service/RedactionChangeLogService.java | 145 +++++++++--------- .../v1/server/RedactionIntegrationTest.java | 17 ++ 10 files changed, 142 insertions(+), 158 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java delete mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java delete mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java new file mode 100644 index 00000000..ebc59a84 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.time.OffsetDateTime; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class Change { + + private ChangeType type; + private OffsetDateTime dateTime; +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java index 0c902a8f..53438d3c 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.redaction.v1.model; public enum ChangeType { - ADDED, REMOVED + ADDED, REMOVED, CHANGED } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java deleted file mode 100644 index 74e385c0..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLog.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.ArrayList; -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class RedactionChangeLog { - - private List redactionLogEntry = new ArrayList<>(); - - private long dictionaryVersion = -1; - private long dossierDictionaryVersion = -1; - private long rulesVersion = -1; - private long legalBasisVersion = -1; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java deleted file mode 100644 index a53d3b0e..00000000 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionChangeLogEntry.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.iqser.red.service.redaction.v1.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.ArrayList; -import java.util.List; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class RedactionChangeLogEntry { - - private String id; - private String type; - private String value; - private String reason; - private int matchedRule; - private String legalBasis; - private boolean redacted; - private boolean isHint; - private boolean isRecommendation; - private String section; - private float[] color; - - @Builder.Default - private List positions = new ArrayList<>(); - private int sectionNumber; - private boolean manual; - private Status status; - private ManualRedactionType manualRedactionType; - private boolean isDictionaryEntry; - - private String textBefore; - private String textAfter; - - @Builder.Default - private List comments = new ArrayList<>(); - - private ChangeType changeType; - - private boolean isDossierDictionaryEntry; - - private boolean excluded; - -} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java new file mode 100644 index 00000000..77fe374d --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class RedactionLogChanges { + + private RedactionLog redactionLog; + private boolean hasChanges; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 38497fa5..dc14eebc 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -58,4 +58,8 @@ public class RedactionLogEntry { private String recategorizationType; private String legalBasisChangeValue; + @EqualsAndHashCode.Exclude + @Builder.Default + private List changes = new ArrayList<>(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java index 2ddbcc46..bd8042ff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeResponseService.java @@ -1,12 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import lombok.NoArgsConstructor; + import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -17,7 +15,7 @@ public class AnalyzeResponseService { private final RedactionServiceSettings redactionServiceSettings; public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, - RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) { + RedactionLog redactionLog, boolean hasUpdates) { boolean hasHints = redactionLog.getRedactionLogEntry() .stream() @@ -41,12 +39,6 @@ public class AnalyzeResponseService { .filter(entry -> !entry.isExcluded()) .anyMatch(entry -> entry.isHint() && entry.getType().equals("image") || entry.isImage()); - boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog - .getRedactionLogEntry() - .isEmpty() && redactionChangeLog.getRedactionLogEntry() - .stream() - .anyMatch(entry -> !entry.getType().equals("false_positive")); - return AnalyzeResult.builder() .dossierId(dossierId) .fileId(fileId) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index e3b37c5f..21df6b0f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -78,9 +78,9 @@ public class ReanalyzeService { log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc .getDictionaryVersion(), analyzeRequest.getDossierTemplateId()); - // first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); - // store redactionLog + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionLog = redactionLogChange.getRedactionLog(); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc .getSectionText())); @@ -88,7 +88,7 @@ public class ReanalyzeService { .getSectionGrid()); long duration = System.currentTimeMillis() - startTime; - return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog); + return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, redactionLogChange.isHasChanges()); } @@ -266,13 +266,13 @@ public class ReanalyzeService { excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); - var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); + var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); long duration = System.currentTimeMillis() - startTime; return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text - .getNumberOfPages(), redactionLog, changeLog); + .getNumberOfPages(), redactionLogChange.getRedactionLog(), redactionLogChange.isHasChanges()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index 1cc743e5..41772291 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -1,19 +1,25 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.ChangeType; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLog; -import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; +import com.iqser.red.service.redaction.v1.model.Change; +import com.iqser.red.service.redaction.v1.model.ChangeType; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogChanges; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -22,76 +28,77 @@ public class RedactionChangeLogService { private final RedactionStorageService redactionStorageService; - public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) { - try { - RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); - var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog); - redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog); - return changeLog; - } catch (Exception e) { - log.debug("Previous redaction log not available"); - return null; - } - - } - - - private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) { + public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) { + RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId); if (previousRedactionLog == null) { - return null; + currentRedactionLog.getRedactionLogEntry().forEach(entry -> { + entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + }); + return new RedactionLogChanges(currentRedactionLog, false); } - List added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry()); - added.removeAll(previousRedactionLog.getRedactionLogEntry()); + List notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> !entry.getChanges() + .get(entry.getChanges().size() - 1) + .getType() + .equals(ChangeType.REMOVED)) + .collect(Collectors.toList()); - List removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry()); + Set added = new HashSet<>(currentRedactionLog.getRedactionLogEntry()); + added.removeAll(notRemovedPreviousEntries); + + Set removed = new HashSet<>(notRemovedPreviousEntries); removed.removeAll(currentRedactionLog.getRedactionLogEntry()); - List changeLogEntries = added.stream() - .map(entry -> convert(entry, ChangeType.ADDED)) - .collect(Collectors.toList()); - changeLogEntries.addAll(removed.stream() - .map(entry -> convert(entry, ChangeType.REMOVED)) - .collect(Collectors.toList())); + Map addedIds = new HashMap<>(); + added.forEach(entry -> { + addedIds.put(entry.getId(), entry); + }); - return new RedactionChangeLog(changeLogEntries, - currentRedactionLog.getDictionaryVersion(), - currentRedactionLog.getDossierDictionaryVersion(), - currentRedactionLog.getRulesVersion(), - currentRedactionLog.getLegalBasisVersion()); - } + Set removedIds = new HashSet<>(); + removed.forEach(entry -> { + removedIds.add(entry.getId()); + }); + List newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry(); - private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) { + List toRemove = new ArrayList<>(); + newRedactionLogEntries.forEach(entry -> { + if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now())); + var newEntry = addedIds.get(entry.getId()); + newEntry.setChanges(changes); + addedIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } else if (removedIds.contains(entry.getId())) { + entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now())); + } else if (addedIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + var newEntry = addedIds.get(entry.getId()); + newEntry.setChanges(changes); + addedIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } + }); - return RedactionChangeLogEntry.builder() - .id(entry.getId()) - .type(entry.getType()) - .value(entry.getValue()) - .reason(entry.getReason()) - .matchedRule(entry.getMatchedRule()) - .legalBasis(entry.getLegalBasis()) - .redacted(entry.isRedacted()) - .isHint(entry.isHint()) - .isRecommendation(entry.isRecommendation()) - .section(entry.getSection()) - .color(entry.getColor()) - .positions(entry.getPositions()) - .sectionNumber(entry.getSectionNumber()) - .manual(entry.isManual()) - .status(entry.getStatus()) - .manualRedactionType(entry.getManualRedactionType()) - .isDictionaryEntry(entry.isDictionaryEntry()) - .textBefore(entry.getTextBefore()) - .textAfter(entry.getTextAfter()) - .comments(entry.getComments()) - .changeType(changeType) - .isDossierDictionaryEntry(entry.isDossierDictionaryEntry()) - .excluded(entry.isExcluded()) - .build(); + newRedactionLogEntries.removeAll(toRemove); + + addedIds.forEach((k, v) -> { + if(v.getChanges().isEmpty()) { + v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now())); + } + newRedactionLogEntries.add(v); + }); + + currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries); + + return new RedactionLogChanges(currentRedactionLog, !addedIds.isEmpty() || !removedIds.isEmpty()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 6949ff40..5a4daeb8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -714,10 +714,13 @@ public class RedactionIntegrationTest { reanlysisVersions.put("physical", 2L); deleted.add("David Chubb"); + deleted.add("mouse"); dictionary.get(FALSE_POSITIVE).add("David Chubb"); reanlysisVersions.put("David Chubb", 3L); + reanlysisVersions.put("mouse", 3L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) @@ -754,6 +757,20 @@ public class RedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } + + deleted.remove("mouse"); + reanlysisVersions.put("mouse", 4L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + reanalyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + System.out.println("hi"); }