Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7384' into 'master'

Browse Source 
More migration fixes

Closes RED-7384

See merge request redactmanager/redaction-service!280
This commit is contained in:
Kilian Schüttler 2024-02-27 10:04:27 +01:00
commit ca3bb600f3
16 changed files with 854 additions and 572 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.93.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.351.0"
val persistenceServiceVersion = "2.359.0"
val springBootStarterVersion = "3.1.5"
configurations {

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java
View File

@ -45,7 +45,7 @@ public class LegacyRedactionLogMergeService {
public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
Set<String> skippedImportedRedactions = new HashSet<>();
log.info("Merging Redaction log with manual redactions");
log.info("Adding manual add Entries and removing skipped or imported entries");
if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
@ -92,6 +92,10 @@ public class LegacyRedactionLogMergeService {
return redactionLog;
}
public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
}
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {

94
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java Normal file
View File

@ -0,0 +1,94 @@
package com.iqser.red.service.redaction.v1.server.migration;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
public class MigrationMapper {
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
toEntityLogType(change.getType()),
change.getDateTime());
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
manualChange.getProcessedDate(),
manualChange.getRequestedDate(),
manualChange.getUserId(),
manualChange.getPropertyChanges());
}
public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
return switch (type) {
case ADDED -> ChangeType.ADDED;
case REMOVED -> ChangeType.REMOVED;
case CHANGED -> ChangeType.CHANGED;
};
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
return switch (manualRedactionType) {
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
};
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
return switch (engine) {
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
};
}
public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
if (entry.getEngines() == null) {
return Collections.emptySet();
}
return entry.getEngines()
.stream()
.map(MigrationMapper::toEntityLogEngine)
.collect(Collectors.toSet());
}
public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
if (manualChanges == null) {
return Collections.emptyList();
}
return manualChanges.stream()
.map(MigrationMapper::toEntityLogManualChanges)
.toList();
}
}

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java
View File

@ -58,19 +58,28 @@ public class MigrationMessageReceiver {
if (redactionLog.getAnalysisVersion() == 0) {
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
} else if (migrationRequest.getManualRedactions() != null) {
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId());
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
migrationRequest.getManualRedactions(),
migrationRequest.getDossierTemplateId());
}
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document, migrationRequest.getDossierTemplateId(), migrationRequest.getManualRedactions());
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
document,
migrationRequest.getDossierTemplateId(),
migrationRequest.getManualRedactions(),
migrationRequest.getFileId());
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}",
migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
migrationRequest.getDossierId(),
migrationRequest.getFileId());
log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
migratedEntityLog.getMigratedIds().getMappings().size(),
migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
migrationRequest.getDossierId(),
migrationRequest.getFileId());
log.info("");
}

135
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java
View File

@ -19,29 +19,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
@ -64,13 +59,16 @@ public class RedactionLogToEntityLogMigrationService {
ManualChangesApplicationService manualChangesApplicationService;
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions) {
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
log.info("Migrating entities for file {}", fileId);
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId);
MigratedIds migratedIds = entitiesToMigrate.stream()
.collect(new MigratedIdsCollector());
applyManualChanges(entitiesToMigrate, manualRedactions);
log.info("applying manual changes to migrated entities for file {}", fileId);
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@ -85,6 +83,8 @@ public class RedactionLogToEntityLogMigrationService {
.toList());
Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
log.info("Writing migrated entities to entityLog for file {}", fileId);
entityLog.setEntityLogEntry(entitiesToMigrate.stream()
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
.toList());
@ -102,6 +102,13 @@ public class RedactionLogToEntityLogMigrationService {
.filter(m -> !m.getOldId().equals(m.getNewId()))
.collect(new MigratedIdsCollector());
List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
.filter(MigrationEntity::needsManualEntry)
.map(MigrationEntity::buildManualRedactionEntry)
.toList();
idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
return new MigratedEntityLog(idsToMigrateInDb, entityLog);
}
@ -117,27 +124,14 @@ public class RedactionLogToEntityLogMigrationService {
manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getLegalBasisChanges(),
manualRedactions.getRecategorizations(),
manualRedactions.getLegalBasisChanges())
manualRedactions.getRecategorizations())
.flatMap(Collection::stream)
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
entitiesToMigrate.forEach(migrationEntity -> manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), Collections.emptyList())
.forEach(manualChange -> {
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migrationEntity.getMigratedEntity() instanceof TextEntity textEntity) {
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
.positions(manualResizeRedaction.getPositions())
.annotationId(migrationEntity.getNewId())
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
.textAfter(manualResizeRedaction.getTextAfter())
.textBefore(manualResizeRedaction.getTextBefore())
.build();
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
} else {
migrationEntity.getMigratedEntity().getManualOverwrite().addChange(manualChange);
}
}));
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
Collections.emptyList()),
manualChangesApplicationService));
}
@ -147,10 +141,10 @@ public class RedactionLogToEntityLogMigrationService {
}
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId) {
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, dossierTemplateId);
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId);
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
return Stream.of(textMigrationEntities.stream(), images.stream())
.flatMap(Function.identity())
.toList();
@ -163,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService {
}
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) {
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
List<Image> images = document.streamAllImages()
.collect(Collectors.toList());
@ -195,7 +189,8 @@ public class RedactionLogToEntityLogMigrationService {
}
String ruleIdentifier;
String reason = Optional.ofNullable(redactionLogImage.getReason()).orElse("");
String reason = Optional.ofNullable(redactionLogImage.getReason())
.orElse("");
if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
ruleIdentifier = "OLDIMG.0.0";
} else {
@ -209,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
} else {
closestImage.skip(ruleIdentifier, reason);
}
migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
}
return migrationEntities;
}
@ -250,40 +245,20 @@ public class RedactionLogToEntityLogMigrationService {
}
private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) {
private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
.stream()
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId)))
.peek(migrationEntity -> {
if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&//
!migrationEntity.getRedactionLogEntry().isHint() &&//
!migrationEntity.getRedactionLogEntry().isRedacted()) {
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (lastManualChangeIsRemove(migrationEntity)) {
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
migrationEntity.getPrecursorEntity()
.skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getPrecursorEntity().isApplied()) {
migrationEntity.getPrecursorEntity()
.apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(),
migrationEntity.getPrecursorEntity().getReason(),
migrationEntity.getPrecursorEntity().getLegalBasis());
} else {
migrationEntity.getPrecursorEntity()
.skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
}
})
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
.toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
entitiesToMigrate.stream()
.map(MigrationEntity::getPrecursorEntity)
.toList());
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
.map(MigrationEntity::getPrecursorEntity)
.toList();
log.info("Finding all possible entities");
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
for (MigrationEntity migrationEntity : entitiesToMigrate) {
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
@ -297,45 +272,19 @@ public class RedactionLogToEntityLogMigrationService {
continue;
}
TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange());
migrationEntity.setMigratedEntity(entity);
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(entity.getId()); // Can only be on one page, since redactionLogEntries can only be on one page
TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
migrationEntity.setMigratedEntity(migratedEntity);
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(migratedEntity.getId());
}
tempEntitiesByValue.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
return entitiesToMigrate;
}
private static boolean lastManualChangeIsRemove(MigrationEntity migrationEntity) {
if (migrationEntity.getRedactionLogEntry().getManualChanges() == null) {
return false;
}
return migrationEntity.getRedactionLogEntry().getManualChanges()
.stream()
.reduce((a, b) -> b)
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
.orElse(false);
}
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node);
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity;
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java
View File

@ -1,7 +1,10 @@
package com.iqser.red.service.redaction.v1.server.model;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import lombok.AllArgsConstructor;
import lombok.Builder;
@ -16,5 +19,4 @@ public class MigratedEntityLog {
MigratedIds migratedIds;
EntityLog entityLog;
}

333
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -1,34 +1,45 @@
package com.iqser.red.service.redaction.v1.server.model;
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data
@Builder
@AllArgsConstructor
@RequiredArgsConstructor
public final class MigrationEntity {
@ -38,28 +49,73 @@ public final class MigrationEntity {
private IEntity migratedEntity;
private String oldId;
private String newId;
private String fileId;
@Builder.Default
List<BaseAnnotation> manualChanges = new LinkedList<>();
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint) {
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
return new MigrationEntity(createPrecursorEntity(redactionLogEntry, hint), redactionLogEntry);
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (redactionLogEntry.lastChangeIsRemoved()) {
precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (lastManualChangeIsRemove(redactionLogEntry)) {
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (precursorEntity.isApplied()) {
precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
} else {
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
}
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
}
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
}
private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.getManualChanges() == null) {
return false;
}
return redactionLogEntry.getManualChanges()
.stream()
.reduce((a, b) -> b)
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
.orElse(false);
}
public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
.stream()
.map(RectangleWithPage::fromRedactionLogRectangle)
.toList();
EntityType entityType = getEntityType(redactionLogEntry, hint);
return PrecursorEntity.builder()
.id(redactionLogEntry.getId())
.value(redactionLogEntry.getValue())
.entityPosition(rectangleWithPages)
.ruleIdentifier(ruleIdentifier)
.reason(Optional.ofNullable(redactionLogEntry.getReason()).orElse(""))
.reason(Optional.ofNullable(redactionLogEntry.getReason())
.orElse(""))
.legalBasis(redactionLogEntry.getLegalBasis())
.type(redactionLogEntry.getType())
.section(redactionLogEntry.getSection())
.engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
.entityType(entityType)
.applied(redactionLogEntry.isRedacted())
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
@ -100,14 +156,6 @@ public final class MigrationEntity {
}
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
toEntityLogType(change.getType()),
change.getDateTime());
}
private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) {
@ -120,42 +168,6 @@ public final class MigrationEntity {
}
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
manualChange.getProcessedDate(),
manualChange.getRequestedDate(),
manualChange.getUserId(),
manualChange.getPropertyChanges());
}
private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
return switch (type) {
case ADDED -> ChangeType.ADDED;
case REMOVED -> ChangeType.REMOVED;
case CHANGED -> ChangeType.CHANGED;
};
}
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
return switch (manualRedactionType) {
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
};
}
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
EntityLogEntry entityLogEntry;
@ -171,10 +183,13 @@ public final class MigrationEntity {
entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
entityLogEntry.setColor(redactionLogEntry.getColor());
entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
entityLogEntry.setChanges(redactionLogEntry.getChanges()
.stream()
.map(MigrationMapper::toEntityLogChanges)
.toList());
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
if (redactionLogEntry.getLegalBasis() != null) {
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
}
@ -198,47 +213,21 @@ public final class MigrationEntity {
}
private List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange> migrateManualChanges(List<ManualChange> manualChanges) {
if (manualChanges == null) {
return Collections.emptyList();
}
return manualChanges.stream().map(MigrationEntity::toEntityLogManualChanges).toList();
}
private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
if (entry.getEngines() == null) {
return Collections.emptySet();
}
return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
}
private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
if (ids == null) {
return Collections.emptySet();
}
return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
}
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
return switch (engine) {
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
};
return ids.stream()
.map(oldToNewIdMapping::get)
.collect(Collectors.toSet());
}
public EntityLogEntry createEntityLogEntry(Image image) {
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
return EntityLogEntry.builder()
return EntityLogEntry.builder()
.id(image.getId())
.value(image.value())
.type(image.type())
@ -249,7 +238,8 @@ public final class MigrationEntity {
.positions(positions)
.containingNodeId(image.getTreeId())
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
.section(redactionLogEntry.getSection())
.section(image.getManualOverwrite().getSection()
.orElse(redactionLogEntry.getSection()))
.textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore())
.imageHasTransparency(image.isTransparent())
@ -270,7 +260,8 @@ public final class MigrationEntity {
.type(precursorEntity.type())
.state(buildEntryState(precursorEntity))
.entryType(buildEntryType(precursorEntity))
.section(redactionLogEntry.getSection())
.section(precursorEntity.getManualOverwrite().getSection()
.orElse(redactionLogEntry.getSection()))
.textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore())
.containingNodeId(Collections.emptyList())
@ -280,12 +271,11 @@ public final class MigrationEntity {
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.startOffset(-1)
.endOffset(-1)
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.positions(precursorEntity.getManualOverwrite().getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.engines(Collections.emptySet())
.build();
}
@ -300,11 +290,13 @@ public final class MigrationEntity {
.positions(rectanglesPerLine)
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.value(entity.getManualOverwrite().getValue()
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.type())
.section(redactionLogEntry.getSection())
.textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore())
.section(entity.getManualOverwrite().getSection()
.orElse(redactionLogEntry.getSection()))
.textAfter(entity.getTextAfter())
.textBefore(entity.getTextBefore())
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@ -322,54 +314,129 @@ public final class MigrationEntity {
private static List<Position> getRectanglesPerLine(TextEntity entity) {
return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
.get(0)
.getRectanglePerLine()
.stream()
.map(rectangle2D -> new Position(rectangle2D, entity.getPositionsOnPagePerPage().get(0).getPage().getNumber()))
.toList());
.get(0).getRectanglePerLine()
.stream()
.map(rectangle2D -> new Position(rectangle2D,
entity.getPositionsOnPagePerPage()
.get(0).getPage().getNumber()))
.toList());
}
private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
return entity.getManualOverwrite().getPositions().map(rects -> rects.stream().map(r -> new Position(r.rectangle2D(), r.pageNumber())).toList());
}
private EntryState buildEntryState(IEntity entity) {
if (entity.applied() && entity.active()) {
return EntryState.APPLIED;
} else if (entity.skipped() && entity.active()) {
return EntryState.SKIPPED;
} else if (entity.ignored()) {
return EntryState.IGNORED;
} else {
return EntryState.REMOVED;
}
}
private EntryType buildEntryType(IEntity entity) {
if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType());
} else if (entity instanceof PrecursorEntity precursorEntity) {
if (precursorEntity.isRectangle()) {
return EntryType.AREA;
}
return getEntryType(precursorEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
}
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
return entity.getManualOverwrite().getPositions()
.map(rects -> rects.stream()
.map(r -> new Position(r.rectangle2D(), r.pageNumber()))
.toList());
}
public boolean hasManualChangesOrComments() {
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty());
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|| hasManualChanges();
}
public boolean hasManualChanges() {
return !manualChanges.isEmpty();
}
public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
manualChanges.addAll(manualChangesToApply);
manualChangesToApply.forEach(manualChange -> {
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
// Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
// Instead, we add an add_locally change to the db.
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
.positions(manualResizeRedaction.getPositions())
.annotationId(getNewId())
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
.textAfter(manualResizeRedaction.getTextAfter())
.textBefore(manualResizeRedaction.getTextBefore())
.build();
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
} else {
migratedEntity.getManualOverwrite().addChange(manualChange);
}
});
}
public ManualRedactionEntry buildManualRedactionEntry() {
assert hasManualChanges();
// currently we need to insert a manual redaction entry, whenever an entity has been resized.
String user = manualChanges.stream()
.filter(mc -> mc instanceof ManualResizeRedaction)
.findFirst()
.orElse(manualChanges.get(0)).getUser();
return ManualRedactionEntry.builder()
.annotationId(newId)
.fileId(fileId)
.type(redactionLogEntry.getType())
.value(redactionLogEntry.getValue())
.reason(redactionLogEntry.getReason())
.legalBasis(redactionLogEntry.getLegalBasis())
.section(redactionLogEntry.getSection())
.addToDictionary(false)
.addToDossierDictionary(false)
.rectangle(false)
.positions(buildPositions(migratedEntity))
.user(user)
.build();
}
private List<Rectangle> buildPositions(IEntity entity) {
if (entity instanceof TextEntity textEntity) {
var positionsOnPage = textEntity.getPositionsOnPagePerPage()
.get(0);
return positionsOnPage.getRectanglePerLine()
.stream()
.map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
.toList();
}
if (entity instanceof PrecursorEntity pEntity) {
return pEntity.getManualOverwrite().getPositions()
.orElse(pEntity.getEntityPosition())
.stream()
.map(p -> new Rectangle((float) p.rectangle2D().getX(),
(float) p.rectangle2D().getY(),
(float) p.rectangle2D().getWidth(),
(float) p.rectangle2D().getHeight(),
p.pageNumber()))
.toList();
}
if (entity instanceof Image image) {
Rectangle2D position = image.getManualOverwrite().getPositions()
.map(p -> p.get(0).rectangle2D())
.orElse(image.getPosition());
return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
} else {
throw new UnsupportedOperationException();
}
}
public boolean needsManualEntry() {
return manualChanges.stream()
.anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
}
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java
View File

@ -43,7 +43,6 @@ public class PrecursorEntity implements IEntity {
String type;
String section;
EntityType entityType;
EntryType entryType;
boolean applied;
boolean isDictionaryEntry;
boolean isDossierDictionaryEntry;
@ -61,8 +60,8 @@ public class PrecursorEntity implements IEntity {
.stream()
.map(RectangleWithPage::fromAnnotationRectangle)
.toList();
var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
var entryType = hint ? EntryType.HINT : (manualRedactionEntry.isRectangle() ? EntryType.AREA : EntryType.ENTITY);
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
manualChangeOverwrite.addChange(manualRedactionEntry);
return PrecursorEntity.builder()
@ -75,7 +74,6 @@ public class PrecursorEntity implements IEntity {
.type(manualRedactionEntry.getType())
.section(manualRedactionEntry.getSection())
.entityType(entityType)
.entryType(entryType)
.applied(true)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
@ -103,7 +101,6 @@ public class PrecursorEntity implements IEntity {
.type(entityLogEntry.getType())
.section(entityLogEntry.getSection())
.entityType(entityType)
.entryType(entityLogEntry.getEntryType())
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
.manualOverwrite(new ManualChangeOverwrite(entityType))
@ -134,7 +131,6 @@ public class PrecursorEntity implements IEntity {
.type(Optional.ofNullable(importedRedaction.getType())
.orElse(IMPORTED_REDACTION_TYPE))
.entityType(entityType)
.entryType(entryType)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java
View File

@ -6,10 +6,22 @@ public enum ImageType {
LOGO,
FORMULA,
SIGNATURE,
OTHER,
OTHER {
@Override
public String toString() {
return "image";
}
},
OCR;
public String toString() {
return name().toLowerCase(Locale.ENGLISH);
}
public static ImageType fromString(String imageType) {
return switch (imageType.toLowerCase(Locale.ROOT)) {

33
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -67,20 +67,19 @@ public class EntityLogCreatorService {
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
return entityLog;
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
dictionaryVersion.getDossierTemplateVersion(),
dictionaryVersion.getDossierVersion(),
rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
}
@ -187,12 +186,11 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
return EntityLogEntry.builder()
.id(image.getId())
.value(image.value())
.type(imageType)
.type(image.type())
.reason(image.buildReasonWithManualChangeDescriptions())
.legalBasis(image.legalBasis())
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
@ -306,7 +304,7 @@ public class EntityLogCreatorService {
}
private EntryState buildEntryState(IEntity entity) {
public static EntryState buildEntryState(IEntity entity) {
if (entity.applied() && entity.active()) {
return EntryState.APPLIED;
@ -320,12 +318,17 @@ public class EntityLogCreatorService {
}
private EntryType buildEntryType(IEntity entity) {
public static EntryType buildEntryType(IEntity entity) {
if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType());
} else if (entity instanceof PrecursorEntity precursorEntity) {
return precursorEntity.getEntryType();
if (precursorEntity.isRectangle()) {
return EntryType.AREA;
}
return getEntryType(precursorEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
}
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
}

138
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java
View File

@ -76,11 +76,19 @@ public class UnprocessedChangesService {
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
.stream()
.map(ManualRedactionEntry::getAnnotationId)
.collect(Collectors.toSet());
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
.stream()
.map(ManualResizeRedaction::getAnnotationId)
.collect(Collectors.toSet());
allAnnotationIds.addAll(resizeIds);
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
.stream()
.toList();
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
@ -99,31 +107,36 @@ public class UnprocessedChangesService {
notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
}
document.getEntities().forEach(textEntity -> {
Set<String> processedIds = new HashSet<>();
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
if (processedIds.contains(positionsOnPerPage.getId())) {
continue;
}
processedIds.add(positionsOnPerPage.getId());
List<Position> positions = positionsOnPerPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
.collect(Collectors.toList());
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
.annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
.textBefore(textEntity.getTextBefore())
.textAfter(textEntity.getTextAfter())
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
.positions(positions)
.build());
}
});
document.getEntities()
.forEach(textEntity -> {
Set<String> processedIds = new HashSet<>();
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
if (processedIds.contains(positionsOnPerPage.getId())) {
continue;
}
processedIds.add(positionsOnPerPage.getId());
List<Position> positions = positionsOnPerPage.getRectanglePerLine()
.stream()
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
.collect(Collectors.toList());
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
.annotationId(allAnnotationIds.stream()
.filter(textEntity::matchesAnnotationId)
.findFirst()
.orElse(""))
.textBefore(textEntity.getTextBefore())
.textAfter(textEntity.getTextAfter())
.section(textEntity.getManualOverwrite().getSection()
.orElse(textEntity.getDeepestFullyContainingNode().toString()))
.positions(positions)
.build());
}
});
notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
}
@ -143,13 +156,13 @@ public class UnprocessedChangesService {
continue;
}
TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get());
TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
.findFirst();
if (optionalManualResizeRedaction.isPresent()) {
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction);
manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
// If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
// from the graph, so it does not get processed and sent back to persistence-service to update its value.
@ -160,60 +173,37 @@ public class UnprocessedChangesService {
}
// remove all temp entities from the graph
tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
tempEntities.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
}
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
correctEntity.setValue(closestEntity.getValue());
correctEntity.setTextAfter(closestEntity.getTextAfter());
correctEntity.setTextBefore(closestEntity.getTextBefore());
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity;
}
return UnprocessedManualEntity.builder()
.annotationId(precursorEntity.getId())
.textAfter("")
.textBefore("")
.section("")
.positions(precursorEntity.getManualOverwrite().getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.build();
}
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
return UnprocessedManualEntity.builder()
.annotationId(precursorEntity.getId())
.textAfter("")
.textBefore("")
.section("")
.positions(precursorEntity.getManualOverwrite()
.getPositions()
.orElse(precursorEntity.getEntityPosition())
.stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList())
.build();
}
private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
return manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.toList();
}
return manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.toList();
}
}

61
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java
View File

@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -51,18 +50,14 @@ public class EntityFromPrecursorCreationService {
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
.stream()
.filter(manualRedactionEntry -> !(idRemovals.stream()
.map(BaseAnnotation::getAnnotationId)
.toList()
.contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate()
.isBefore(idRemovals.stream()
.filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
.findFirst()
.get()
.getRequestDate())))
.filter(manualRedactionEntry -> idRemovals.stream()
.filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
.filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
.findAny()//
.isEmpty())
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.map(manualRedactionEntry -> //
PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> {
if (manualEntity.getEntityType().equals(EntityType.HINT)) {
manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
@ -90,8 +85,14 @@ public class EntityFromPrecursorCreationService {
public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList());
var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList();
var notFoundEntities = precursorEntities.stream()
.filter(PrecursorEntity::isRectangle)
.collect(Collectors.toList());
var findableEntities = precursorEntities.stream()
.filter(precursorEntity -> !precursorEntity.isRectangle())
.toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
for (PrecursorEntity precursorEntity : findableEntities) {
@ -102,7 +103,12 @@ public class EntityFromPrecursorCreationService {
}
createCorrectEntity(precursorEntity, optionalClosestEntity.get());
}
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
tempEntitiesByValue.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
return notFoundEntities;
}
@ -113,9 +119,23 @@ public class EntityFromPrecursorCreationService {
* @param precursorEntity The entity identifier for the RedactionEntity.
* @param closestEntity The closest Boundary to the RedactionEntity.
*/
private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
return createCorrectEntity(precursorEntity, closestEntity, false);
}
public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
TextEntity correctEntity;
if (generateId) {
correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
precursorEntity.type(),
precursorEntity.getEntityType(),
closestEntity.getDeepestFullyContainingNode());
} else {
correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
}
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
@ -125,14 +145,17 @@ public class EntityFromPrecursorCreationService {
correctEntity.setTextAfter(closestEntity.getTextAfter());
correctEntity.setTextBefore(closestEntity.getTextBefore());
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.getIntersectingNodes()
.forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages()
.forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addEngines(precursorEntity.getEngines());
return correctEntity;
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Set;
import java.util.function.BiConsumer;
@ -17,7 +18,7 @@ public class MigratedIdsCollector implements Collector<MigrationEntity, Migrated
@Override
public Supplier<MigratedIds> supplier() {
return () -> new MigratedIds(new LinkedList<>());
return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
}

34
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java
View File

@ -31,6 +31,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@ -49,7 +50,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@ -107,7 +107,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
@SneakyThrows
public void testSave() {
MigratedIds ids = new MigratedIds(new LinkedList<>());
MigratedIds ids = new MigratedIds(new LinkedList<>(), null);
ids.addMapping("123", "321");
ids.addMapping("123", "321");
ids.addMapping("123", "321");
@ -173,7 +173,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
mergedRedactionLog = redactionLog;
}
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog, document, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog,
document,
TEST_DOSSIER_TEMPLATE_ID,
manualRedactions,
TEST_FILE_ID);
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
@ -187,10 +191,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
// assertEquals(legacyRedactionLogMergeService.getNumberOfAffectedAnnotations(manualRedactions), migratedIds.size());
migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds));
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID)
.build());
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
@ -268,13 +273,24 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
if (!redactionLogEntry.isImage()) {
assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
}
if (entityLogEntry.getManualChanges()
.stream()
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) {
assertEquals(redactionLogEntry.getType(), entityLogEntry.getType());
}
assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size());
assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
// assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
// assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
if (entityLogEntry.getManualChanges()
.stream()
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RESIZE) || mc.getManualRedactionType().equals(ManualRedactionType.RESIZE_IN_DICTIONARY))) {
assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
}
if (entityLogEntry.getManualChanges()
.stream()
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || mc.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT))) {
assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
}
assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());

539
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -119,15 +119,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);
@ -169,9 +169,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
entityLog.getEntityLogEntry()
.forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
@ -216,12 +217,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ManualRedactions manualRedactions = ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.annotationId("c6be5277f5ee60dc3d83527798b7fe02")
.value("Dr. Alan")
.positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.build()))
.annotationId("c6be5277f5ee60dc3d83527798b7fe02")
.value("Dr. Alan")
.positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
.requestDate(OffsetDateTime.now())
.updateDictionary(false)
.build()))
.build();
request.setManualRedactions(manualRedactions);
@ -256,7 +257,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
var values = redactionLog.getEntityLogEntry()
.stream()
.map(EntityLogEntry::getValue)
.collect(Collectors.toList());
assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
}
@ -268,8 +272,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
"files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactionClasspathResource.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactionClasspathResource.getInputStream());
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
System.out.println("Start Full integration test");
@ -353,10 +357,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry()
.stream()
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
.findAny()
.get();
assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry()
.stream()
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
.findAny()
.get();
assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
}
@ -386,9 +398,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
entityLog.getEntityLogEntry()
.forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1));
@ -421,11 +434,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
AnalyzeRequest request = uploadFileToStorage(fileName);
request.setFileAttributes(List.of(FileAttribute.builder()
.id("fileAttributeId")
.label("Vertebrate Study")
.placeholder("{fileattributes.vertebrateStudy}")
.value("true")
.build()));
.id("fileAttributeId")
.label("Vertebrate Study")
.placeholder("{fileattributes.vertebrateStudy}")
.value("true")
.build()));
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
@ -449,7 +462,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
correctFound++;
continue loop;
}
if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) {
if (Objects.equals(entityLogEntry.getContainingNodeId()
.get(0),
section.getTreeId()
.get(0))) {
String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
correctFound++;
@ -481,12 +497,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
.value("Redact")
.addToDictionary(true)
.addToDossierDictionary(true)
.positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2)))
.type("dossier_redaction")
.build()));
.value("Redact")
.addToDictionary(true)
.addToDossierDictionary(true)
.positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2)))
.type("dossier_redaction")
.build()));
request.setManualRedactions(manualRedactions);
@ -548,7 +564,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
var changes = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
.findFirst()
.get().getChanges();
assertThat(changes.size()).isEqualTo(2);
@ -568,18 +588,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream());
long start = System.currentTimeMillis();
AnalyzeRequest request = uploadFileToStorage(fileName);
request.setFileAttributes(List.of(FileAttribute.builder()
.id("fileAttributeId")
.label("Vertebrate Study")
.placeholder("{fileattributes.vertebrateStudy}")
.value("true")
.build()));
.id("fileAttributeId")
.label("Vertebrate Study")
.placeholder("{fileattributes.vertebrateStudy}")
.value("true")
.build()));
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
@ -601,7 +621,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
.map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
.toList();
List<String> valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList();
List<String> valuesInRedactionLog = entityLog.getEntityLogEntry()
.stream()
.filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
.map(EntityLogEntry::getValue)
.toList();
assertEquals(valuesInRedactionLog, valuesInDocument);
@ -628,11 +652,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.setRecategorizations(Set.of(ManualRecategorization.builder()
.annotationId("37eee3e9d589a5cc529bfec38c3ba479")
.fileId("fileId")
.type("signature")
.requestDate(OffsetDateTime.now())
.build()));
.annotationId("37eee3e9d589a5cc529bfec38c3ba479")
.fileId("fileId")
.type("signature")
.requestDate(OffsetDateTime.now())
.build()));
request.setManualRedactions(manualRedactions);
@ -683,40 +707,40 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getIdsToRemove()
.add(IdRemoval.builder()
.annotationId("308dab9015bfafd911568cffe0a7f7de")
.fileId(TEST_FILE_ID)
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
.build());
.annotationId("308dab9015bfafd911568cffe0a7f7de")
.fileId(TEST_FILE_ID)
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
.build());
manualRedactions.getForceRedactions()
.add(ManualForceRedaction.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.legalBasis("Something")
.requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
.build());
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.legalBasis("Something")
.requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
.build());
manualRedactions.getIdsToRemove()
.add(IdRemoval.builder()
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
.build());
.annotationId("0b56ea1a87c83f351df177315af94f0d")
.fileId(TEST_FILE_ID)
.requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
.processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
.build());
request.setManualRedactions(manualRedactions);
AnalyzeResult result = analyzeService.analyze(request);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions)
.colors(colors)
.types(types)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
.manualRedactions(manualRedactions)
.colors(colors)
.types(types)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
@ -932,12 +956,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.manualRedactions(manualRedactions)
.colors(colors)
.types(types)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
.manualRedactions(manualRedactions)
.colors(colors)
.types(types)
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
@ -960,15 +984,16 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getEntityLogEntry().forEach(entry -> {
if (!entry.getEntryType().equals(EntryType.HINT)) {
if (entry.getType().equals("CBI_author")) {
assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
} else if (entry.getType().equals("CBI_address")) {
assertThat(entry.getReason()).isEqualTo("No vertebrate found");
}
}
});
redactionLog.getEntityLogEntry()
.forEach(entry -> {
if (!entry.getEntryType().equals(EntryType.HINT)) {
if (entry.getType().equals("CBI_author")) {
assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
} else if (entry.getType().equals("CBI_address")) {
assertThat(entry.getReason()).isEqualTo("No vertebrate found");
}
}
});
}
@ -1005,18 +1030,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
String manualAddId = UUID.randomUUID().toString();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build()));
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
@ -1027,7 +1052,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
@ -1038,39 +1063,63 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build())));
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build())));
manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder()
.annotationId("fc287b74be2421156ab2895c7474ccdd")
.fileId("fileId")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
.positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(),
Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(),
Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
.updateDictionary(false)
.build()));
.annotationId("fc287b74be2421156ab2895c7474ccdd")
.fileId("fileId")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
.positions(List.of(Rectangle.builder()
.topLeftX(289.44595f)
.topLeftY(327.567f)
.width(7.648041f)
.height(82.51475f)
.page(1)
.build(),
Rectangle.builder()
.topLeftX(298.67056f)
.topLeftY(327.567f)
.width(7.648041f)
.height(75.32377f)
.page(1)
.build(),
Rectangle.builder()
.topLeftX(307.89517f)
.topLeftY(327.567f)
.width(7.648041f)
.height(61.670967f)
.page(1)
.build(),
Rectangle.builder()
.topLeftX(316.99985f)
.topLeftY(327.567f)
.width(7.648041f)
.height(38.104286f)
.page(1)
.build()))
.updateDictionary(false)
.build()));
analyzeService.reanalyze(request);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.manualRedactions(manualRedactions)
.build());
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.manualRedactions(manualRedactions)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
@ -1110,8 +1159,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactions.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactions.getInputStream());
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
@ -1124,17 +1173,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument());
}
entityLog.getEntityLogEntry().forEach(entry -> {
if (entry.getValue() == null) {
return;
}
if (entry.getValue().equals("David")) {
assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
}
if (entry.getValue().equals("annotation")) {
assertThat(entry.getImportedRedactionIntersections()).isEmpty();
}
});
entityLog.getEntityLogEntry()
.forEach(entry -> {
if (entry.getValue() == null) {
return;
}
if (entry.getValue().equals("David")) {
assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
}
if (entry.getValue().equals("annotation")) {
assertThat(entry.getImportedRedactionIntersections()).isEmpty();
}
});
}
@ -1163,7 +1213,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
}
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
var values = entityLog.getEntityLogEntry()
.stream()
.map(EntityLogEntry::getValue)
.collect(Collectors.toList());
assertThat(values).contains("Mrs. Robinson");
assertThat(values).contains("Mr. Bojangles");
@ -1178,8 +1231,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@ -1188,23 +1241,26 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis");
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.build());
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.build());
analyzeService.reanalyze(request);
System.out.println("Finished reanalysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> {
assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
});
entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getType().equals("signature"))
.forEach(entityLogEntry -> {
assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
});
}
@ -1215,8 +1271,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@ -1225,22 +1281,22 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis");
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.type("CBI_author")
.legalBasis("")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.build()))
.build());
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.type("CBI_author")
.legalBasis("")
.requestDate(OffsetDateTime.now())
.fileId(TEST_FILE_ID)
.build()))
.build());
analyzeService.reanalyze(request);
System.out.println("Finished reanalysis");
@ -1267,11 +1323,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
String manualAddId2 = UUID.randomUUID().toString();
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build());
ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId,
positions,
"the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
positions,
"the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2,
positions,
"commercial information revealing sourcing, market shares or business strategy of the applicant");
positions,
"commercial information revealing sourcing, market shares or business strategy of the applicant");
IdRemoval idRemoval = getIdRemoval(manualAddId);
IdRemoval idRemoval2 = getIdRemoval(manualAddId2);
@ -1283,53 +1339,98 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.APPLIED);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.APPLIED);
request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
.idsToRemove(Set.of(idRemoval, idRemoval2))
.build());
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
.idsToRemove(Set.of(idRemoval, idRemoval2))
.build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.REMOVED);
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
.idsToRemove(Set.of(idRemoval, idRemoval2))
.build());
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval2)).build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.APPLIED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.REMOVED);
}
@ -1344,21 +1445,35 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
analyzeService.analyze(request);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get();
var david = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getValue().equals("David"))
.findFirst()
.get();
request.setManualRedactions(ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.updateDictionary(true)
.annotationId(david.getId())
.requestDate(OffsetDateTime.now())
.value("David Ksenia")
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build()))
.addToAllDossiers(false)
.build()))
.build());
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.updateDictionary(true)
.annotationId(david.getId())
.requestDate(OffsetDateTime.now())
.value("David Ksenia")
.positions(List.of(Rectangle.builder()
.topLeftX(56.8f)
.topLeftY(293.564f)
.width(65.592f)
.height(15.408f)
.page(1)
.build()))
.addToAllDossiers(false)
.build()))
.build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get();
var resizedEntity = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getId().equals(david.getId()))
.findFirst()
.get();
assertEquals(resizedEntity.getState(), EntryState.APPLIED);
assertEquals(resizedEntity.getValue(), "David Ksenia");
}

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java
View File

@ -34,7 +34,8 @@ public class LayoutParsingRequestProvider {
.textBlockFileStorageId(textBlockFileStorageId)
.positionBlockFileStorageId(positionBlockFileStorageId)
.pageFileStorageId(pageFileStorageId)
.simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
.simplifiedTextStorageId(simplifiedTextStorageId)
.viewerDocumentStorageId(viewerDocumentStorageId)
.visualLayoutParsingFileId(Optional.empty())
.build();
}