Merge branch 'RED-7834' into 'master'

RED-7834: fixes for migration code

Closes RED-7834

See merge request redactmanager/redaction-service!220
This commit is contained in:
Kilian Schüttler 2023-12-08 11:05:49 +01:00
commit efc01f9d20
18 changed files with 562 additions and 46 deletions

View File

@ -1,5 +1,8 @@
package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -11,7 +14,9 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
public class MigrationRequest {
String dossierTemplateId;
String dossierId;
String fileId;
ManualRedactions manualRedactions;
}

View File

@ -0,0 +1,406 @@
package com.iqser.red.service.redaction.v1.server.migration;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class LegacyRedactionLogMergeService {
private final DictionaryService dictionaryService;
public RedactionLog mergeManualChanges(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
var skippedImportedRedactions = new HashSet<>();
log.info("Merging Redaction log with manual redactions");
if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, dossierTemplateId);
if (entry.isImported() && !entry.isRedacted()) {
skippedImportedRedactions.add(entry.getId());
}
}
}
Set<String> processedIds = new HashSet<>();
redactionLog.getRedactionLogEntry().removeIf(entry -> {
if (entry.getImportedRedactionIntersections() != null) {
entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
.equals("ocr")))) {
return true;
}
}
if (processedIds.contains(entry.getId())) {
log.info("Duplicate annotation found with id {}", entry.getId());
return true;
}
processedIds.add(entry.getId());
return false;
});
return redactionLog;
}
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
manualRedactions.getRecategorizations().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getIdsToRemove().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getForceRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getLegalBasisChanges().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
manualRedactions.getResizeRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
Collections.sort(manualRedactionWrappers);
return manualRedactionWrappers;
}
private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, String dossierTemplateId) {
manualRedactionWrappers.forEach(mrw -> {
Object item = mrw.getItem();
if (item instanceof ManualRecategorization imageRecategorization) {
processManualImageRecategorization(redactionLogEntry, dossierTemplateId, imageRecategorization);
}
if (item instanceof IdRemoval manualRemoval) {
processIdRemoval(redactionLogEntry, manualRemoval);
}
if (item instanceof ManualForceRedaction manualForceRedact) {
processManualForceRedaction(redactionLogEntry, dossierTemplateId, manualForceRedact);
}
if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
}
if (item instanceof ManualResizeRedaction manualResizeRedact) {
processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
}
});
}
private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualRecategorization imageRecategorization) {
String manualOverrideReason = null;
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setType(imageRecategorization.getType());
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setHint(true);
} else {
redactionLogEntry.setHint(false);
}
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
redactionLogEntry.getManualChanges()
.add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
}
private String mergeReasonIfNecessary(String currentReason, String addition) {
if (currentReason != null) {
if (!currentReason.contains(addition)) {
return currentReason + addition;
}
return currentReason;
} else {
return "";
}
}
private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
log.debug("Skipping merge for dictionary-modifying entry");
} else {
String manualOverrideReason = null;
if (isApprovedRedaction) {
redactionLogEntry.setRedacted(false);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setHint(false);
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
}
redactionLogEntry.getManualChanges()
.add(ManualChange.from(manualRemoval)
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
}
private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
}
private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualForceRedaction manualForceRedact) {
String manualOverrideReason = null;
var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
// Forcing a skipped hint should result in a hint
if (dictionaryIsHint) {
redactionLogEntry.setHint(true);
} else {
redactionLogEntry.setRedacted(true);
}
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
redactionLogEntry.getManualChanges().add(manualChange);
}
private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
redactionLogEntry.setRedacted(true);
if (manualLegalBasisChange.getSection() != null) {
redactionLogEntry.setSection(manualLegalBasisChange.getSection());
}
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
redactionLogEntry.setValue(manualLegalBasisChange.getValue());
}
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
}
if (manualOverrideReason != null) {
redactionLogEntry.setReason(manualOverrideReason);
}
var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
if (manualLegalBasisChange.getSection() != null) {
manualChange.withChange("section", manualLegalBasisChange.getSection());
}
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
manualChange.withChange("value", manualLegalBasisChange.getValue());
}
redactionLogEntry.getManualChanges().add(manualChange);
}
private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
redactionLogEntry.setValue(manualResizeRedact.getValue());
}
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
}
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
}
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.getManualChanges()
.add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
}
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (shouldCreateManualEntry(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
redactionLogEntries.add(redactionLogEntry);
}
}
return redactionLogEntries;
}
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle> positions) {
return positions.stream()
.map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
.collect(Collectors.toList());
}
@SuppressWarnings("PMD.UselessParentheses")
private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
List<ManualChange> changeList = new ArrayList<>();
changeList.add(change);
return RedactionLogEntry.builder()
.id(id)
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.sourceId(manualRedactionEntry.getSourceId())
.section(manualRedactionEntry.getSection())
.type(manualRedactionEntry.getType())
.redacted(true)
.isHint(false)
.sectionNumber(-1)
.rectangle(manualRedactionEntry.isRectangle())
.manualChanges(changeList)
.changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
.build();
}
@Data
@AllArgsConstructor
private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
private String id;
private OffsetDateTime date;
private Object item;
@Override
public int compareTo(ManualRedactionWrapper o) {
return this.date.compareTo(o.date);
}
}
}

View File

@ -0,0 +1,70 @@
package com.iqser.red.service.redaction.v1.server.migration;
import java.lang.reflect.Field;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import lombok.SneakyThrows;
@Service
@SuppressWarnings("PMD")
public class LegacyVersion0MigrationService {
public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
List<RedactionLogEntry> mergedEntries = new LinkedList<>();
Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId));
for (List<RedactionLogEntry> entries : entriesById.values()) {
if (entries.isEmpty()) {
continue;
}
if (entries.size() == 1) {
mergedEntries.add(entries.get(0));
continue;
}
List<RedactionLogEntry> sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList();
RedactionLogEntry initialEntry = sortedEntries.get(0);
for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {
copyNonNullFields(entry, initialEntry);
}
mergedEntries.add(initialEntry);
}
redactionLog.setRedactionLogEntry(mergedEntries);
return redactionLog;
}
@SneakyThrows
public static void copyNonNullFields(RedactionLogEntry source, RedactionLogEntry destination) {
if (source == null || destination == null) {
throw new IllegalArgumentException("Source and destination objects must not be null");
}
Class<?> sourceClass = source.getClass();
Field[] sourceFields = sourceClass.getDeclaredFields();
for (Field field : sourceFields) {
field.setAccessible(true);
Object value = field.get(source);
if (value != null) {
field.set(destination, value);
}
}
}
}

View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.queue;
package com.iqser.red.service.redaction.v1.server.migration;
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
@ -16,7 +16,6 @@ import com.iqser.red.service.redaction.v1.model.MigrationRequest;
import com.iqser.red.service.redaction.v1.model.MigrationResponse;
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@ -35,6 +34,8 @@ public class MigrationMessageReceiver {
ObjectMapper objectMapper;
RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
RedactionStorageService redactionStorageService;
LegacyRedactionLogMergeService legacyRedactionLogMergeService;
LegacyVersion0MigrationService legacyVersion0MigrationService;
RabbitTemplate rabbitTemplate;
@ -45,8 +46,16 @@ public class MigrationMessageReceiver {
MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
// TODO: if an image is not found, try to copy the old one exactly (like with TextEntities)
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
if (redactionLog.getAnalysisVersion() == 0) {
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
} else if (migrationRequest.getManualRedactions() != null) {
redactionLog = legacyRedactionLogMergeService.mergeManualChanges(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId());
}
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);

View File

@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@ -51,7 +52,7 @@ public final class MigrationEntity {
String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
EntityType entityType = getEntityType(redactionLogEntry);
return ManualEntity.builder()
.id(redactionLogEntry.getId())
.value(redactionLogEntry.getValue())
@ -61,11 +62,12 @@ public final class MigrationEntity {
.legalBasis(redactionLogEntry.getLegalBasis())
.type(redactionLogEntry.getType())
.section(redactionLogEntry.getSection())
.entityType(getEntityType(redactionLogEntry))
.entityType(entityType)
.applied(redactionLogEntry.isRedacted())
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
.rectangle(redactionLogEntry.isRectangle())
.manualOverwrite(new ManualChangeOverwrite(entityType))
.build();
}

View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.drools;
import java.util.regex.Pattern;
public record RuleType(String name) {
static Pattern alphaNumeric = Pattern.compile("^[a-zA-Z0-9]+$");

View File

@ -60,7 +60,9 @@ public class EntityLogCreatorService {
}
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, Document document, List<ManualEntity> notFoundManualEntities,
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document,
List<ManualEntity> notFoundManualEntities,
DictionaryVersion dictionaryVersion,
long rulesVersion) {
@ -124,8 +126,8 @@ public class EntityLogCreatorService {
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> !entry.getType()
.equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
.filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId()
.isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
.toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
@ -176,7 +178,9 @@ public class EntityLogCreatorService {
EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
entityLogEntries.setId(positionOnPage.getId());
List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
.toList();
entityLogEntries.setPositions(rectanglesPerLine);
@ -207,7 +211,8 @@ public class EntityLogCreatorService {
.section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
.imageHasTransparency(image.isTransparent())
.manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(image)).entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.state(buildEntryState(image))
.entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.build();
}

View File

@ -95,7 +95,7 @@ public class RedactionStorageService {
RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
RedactionLog.class);
redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !(entry.getValue() == null || entry.getValue().isEmpty())).collect(Collectors.toList()));
return redactionLog;
} catch (StorageObjectDoesNotExist e) {
log.debug("RedactionLog not available.");

View File

@ -39,11 +39,11 @@ import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentInt
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
import com.iqser.red.service.redaction.v1.server.migration.RedactionLogToEntityLogMigrationService;
import lombok.SneakyThrows;
@Disabled
//@Disabled
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(MigrationIntegrationTest.TestConfiguration.class)
@ -72,17 +72,20 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
assert ids2.getMappings().size() == 5;
}
@Test
@SneakyThrows
public void testMigration() {
String fileName = "files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf";
String imageFileName = "files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json";
String tableFileName = "files/migration/def8f960580f088b975ba806dfae1f87.TABLES.json";
String filesPrefix = "files/migration/def8f960580f088b975ba806dfae1f87";
String fileName = filesPrefix + ".ORIGIN.pdf";
String imageFileName = filesPrefix + ".IMAGE_INFO.json";
String tableFileName = filesPrefix + ".TABLES.json";
Document document = buildGraph(fileName, imageFileName, tableFileName);
RedactionLog redactionLog;
try (var in = new ClassPathResource("files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json").getInputStream()) {
try (var in = new ClassPathResource(filesPrefix + ".REDACTION_LOG.json").getInputStream()) {
redactionLog = mapper.readValue(in, RedactionLog.class);
}
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
@ -129,30 +132,34 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
assertEquals(redactionLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()),
entityLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()));
if (redactionLogEntry.getEngines() == null) {
assertTrue(entityLogEntry.getEngines().isEmpty());
} else {
assertEquals(redactionLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()),
entityLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()));
}
}
private boolean positionsAlmostEqual(List<Rectangle> positions1, List<Position> positions2) {
double tolerance = 2;
double tolerance = 3;
for (int i = 0; i < positions1.size(); i++) {
Rectangle p1 = positions1.get(0);
Position p2 = positions2.get(0);
if (p1.getPage() != p2.getPageNumber()) {
return false;
}
if (Math.abs(p1.getHeight() - p2.getRectangle()[3]) > tolerance) {
if (Math.abs(p1.getHeight() - p2.h()) > tolerance) {
return false;
}
if (Math.abs(p1.getWidth() - p2.getRectangle()[2]) > tolerance) {
if (Math.abs(p1.getWidth() - p2.w()) > tolerance) {
return false;
}
if (Math.abs(p1.getTopLeft().getX() - p2.getRectangle()[0]) > tolerance) {
if (Math.abs(p1.getTopLeft().getX() - p2.x()) > tolerance) {
return false;
}
if (Math.abs(p1.getTopLeft().getY() - p2.getRectangle()[1]) > tolerance) {
if (Math.abs(p1.getTopLeft().getY() - p2.y()) > tolerance) {
return false;
}
}

View File

@ -1,5 +1,6 @@
plugins {
application
id("com.iqser.red.service.java-conventions")
}
group = "com.knecon.fforesight.utility"

View File

@ -26,6 +26,7 @@ import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
import lombok.SneakyThrows;
@SuppressWarnings("PMD")
public class Main {
public static void main(String[] args) {
@ -123,7 +124,7 @@ public class Main {
private static String escapeRuleStringIfOptionIsSet(CommandLine finalCmd, String ruleFileString) {
if (finalCmd.hasOption("e")) {
ruleFileString = RuleFileIO.escapeAndWrap(ruleFileString);
return RuleFileIO.escapeAndWrap(ruleFileString);
}
return ruleFileString;
}
@ -150,6 +151,7 @@ public class Main {
.collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), RuleFileParser::parseRuleIdentifiersFromFile));
}
private static ApplicationType getApplicationType(final CommandLine cmd) {
String application = cmd.hasOption("a") ? cmd.getOptionValue("a") : "RM";

View File

@ -9,6 +9,7 @@ import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
import lombok.SneakyThrows;
@SuppressWarnings("PMD")
public class RuleManagementResources {
public static InputStream getAllRulesInputStream(ApplicationType applicationType) {

View File

@ -96,7 +96,7 @@ public class RuleFileParser {
/**
* Creates a BluePrint from all redact manager, documine and component rule files
* Creates a BluePrint from all redact manager, documine and component rule files.
*
* @return RuleFileBluePrint containing all rules from any all rule files
*/

View File

@ -19,6 +19,7 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
return fromString(identifier);
}
public static RuleIdentifier fromRuleType(RuleType ruleType) {
return new RuleIdentifier(ruleType, null, null);
@ -26,6 +27,7 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
public static RuleIdentifier fromString(String identifier) {
String[] values = identifier.split("\\.");
RuleType type = RuleType.fromString(values[0]);
Integer group = values.length > 1 ? parseIntOrStar(values[1]) : null;
@ -33,6 +35,7 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
return new RuleIdentifier(type, group, id);
}
public static Set<RuleIdentifier> fromListOfIdentifiersString(String input) {
return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet());
@ -40,8 +43,10 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
private static Integer parseIntOrStar(String value) {
value = value.replaceAll("\r","");
return !value.equals("*") ? Integer.parseInt(value) : null;
String cleanedValue = value;
cleanedValue = cleanedValue.replaceAll("\r", "");
return !cleanedValue.equals("*") ? Integer.parseInt(cleanedValue) : null;
}

View File

@ -28,8 +28,8 @@ public class RuleType {
public static RuleType fromString(String value) {
value = value.replaceAll("\r", "");
return new RuleType(value);
String cleanedValue = value.replaceAll("\r", "");
return new RuleType(cleanedValue);
}

View File

@ -39,7 +39,7 @@ import lombok.experimental.UtilityClass;
@UtilityClass
public class OldRulesParser {
final static List<String> HEADERS = List.of("id", "old rule names", "old rule code", "translates to");
static List<String> HEADERS = List.of("id", "old rule names", "old rule code", "translates to");
@SneakyThrows
@ -117,7 +117,7 @@ public class OldRulesParser {
StringWriter sw = new StringWriter();
CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS.toArray(String[]::new)).build();
try (final CSVPrinter printer = new CSVPrinter(sw, csvFormat)) {
try (CSVPrinter printer = new CSVPrinter(sw, csvFormat)) {
for (OldRulesCsvRecord record : records) {
printer.printRecord(Stream.of(record.id, RuleFileIO.escapeAndWrap(record.names), RuleFileIO.escapeAndWrap(record.code), record.translatesTo));
}
@ -162,17 +162,18 @@ public class OldRulesParser {
private List<RuleIdentifier> parseRuleIdentifiers(String value) {
if (value.startsWith("[")) {
value = value.substring(1);
String cleanedValue = value;
if (cleanedValue.startsWith("[")) {
cleanedValue = cleanedValue.substring(1);
}
if (value.endsWith("]")) {
value = value.substring(0, value.length() - 1);
if (cleanedValue.endsWith("]")) {
cleanedValue = cleanedValue.substring(0, value.length() - 1);
}
if (value.isEmpty() || value.isBlank()) {
if (cleanedValue.isEmpty() || cleanedValue.isBlank()) {
return Collections.emptyList();
}
List<RuleIdentifier> ruleIdentifiers = new LinkedList<>();
for (String identifier : value.split(", ")) {
for (String identifier : cleanedValue.split(", ")) {
ruleIdentifiers.add(RuleIdentifier.fromString(identifier));
}
return ruleIdentifiers;

View File

@ -20,9 +20,11 @@ public final class RuleFileIO {
if (rulesString.isBlank() || rulesString.isEmpty()) {
return rulesString;
}
rulesString = rulesString.substring(1, rulesString.length() - 1);
rulesString = rulesString.translateEscapes();
return rulesString;
String ruleStringToUnescape = rulesString;
ruleStringToUnescape = ruleStringToUnescape.substring(1, ruleStringToUnescape.length() - 1);
ruleStringToUnescape = ruleStringToUnescape.translateEscapes();
return ruleStringToUnescape;
}
@ -43,9 +45,10 @@ public final class RuleFileIO {
public String escapeAndWrap(String rulesString) {
rulesString = StringEscapeUtils.escapeJava(rulesString);
rulesString = "\"" + rulesString + "\"";
return rulesString;
String rulesStringToEscape = rulesString;
rulesStringToEscape = StringEscapeUtils.escapeJava(rulesStringToEscape);
rulesStringToEscape = "\"" + rulesStringToEscape + "\"";
return rulesStringToEscape;
}