This commit is contained in:
maverickstuder 2024-03-14 10:04:47 +01:00
parent f6d169db43
commit 99fdc9702d
25 changed files with 287 additions and 150 deletions

View File

@ -70,6 +70,7 @@ public class MigrationMessageReceiver {
migrationRequest.getFileId());
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
//todo
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());

View File

@ -227,7 +227,7 @@ public final class MigrationEntity {
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
return EntityLogEntry.builder()
.entryId(image.getId())
.id(image.getId())
.value(image.value())
.type(image.type())
.reason(image.buildReasonWithManualChangeDescriptions())
@ -252,7 +252,7 @@ public final class MigrationEntity {
public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
return EntityLogEntry.builder()
.entryId(precursorEntity.getId())
.id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())
@ -285,7 +285,7 @@ public final class MigrationEntity {
assert entity.getPositionsOnPagePerPage().size() == 1;
List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
return EntityLogEntry.builder()
.entryId(entity.getId())
.id(entity.getId())
.positions(rectanglesPerLine)
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())

View File

@ -92,7 +92,7 @@ public class PrecursorEntity implements IEntity {
.toList();
EntityType entityType = getEntityType(entityLogEntry.getEntryType());
return PrecursorEntity.builder()
.id(entityLogEntry.getEntryId())
.id(entityLogEntry.getId())
.value(entityLogEntry.getValue())
.entityPosition(rectangleWithPages)
.ruleIdentifier(entityLogEntry.getMatchedRule())

View File

@ -71,7 +71,7 @@ public class Entity {
public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
return Entity.builder()
.id(e.getEntryId())
.id(e.getId())
.type(e.getType())
.entryType(e.getEntryType())
.state(e.getState())

View File

@ -4,15 +4,11 @@ import java.lang.reflect.Field;
import java.lang.reflect.InaccessibleObjectException;
import java.util.Collection;
import org.bouncycastle.util.Iterable;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.util.ReflectionUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.CascadeSave;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;

View File

@ -0,0 +1,12 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
public @interface CascadeSave {
//
}

View File

@ -1,33 +1,41 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.ArrayList;
import java.util.List;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Document(collection = "entity-logs")
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
private String id;
private EntityLog entityLog;
private String dossierId;
private String fileId;
/**
private long analysisVersion;
private int analysisNumber;
private List<EntityLogEntry> entityLogEntry = new ArrayList<>();
@DBRef
@CascadeSave
private List<EntityLogEntryDocument> entityLogEntryDocument = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
@ -36,19 +44,29 @@ public class EntityLogDocument {
private long legalBasisVersion = -1;
public EntityLogDocument(String id, EntityLog entityLog) {
this.id = id;
public EntityLogDocument(String dossierId, String fileId, EntityLog entityLog) {
this.id = getDocumentId(dossierId, fileId);
this.dossierId = dossierId;
this.fileId = fileId;
this.analysisVersion = entityLog.getAnalysisVersion();
this.analysisNumber = entityLog.getAnalysisNumber();
this.entityLogEntry = entityLog.getEntityLogEntry();
this.entityLogEntryDocument = new ArrayList<>(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(this.id, entityLogEntry)).toList());
this.legalBasis = entityLog.getLegalBasis();
this.dictionaryVersion = entityLog.getDictionaryVersion();
this.dossierDictionaryVersion = entityLog.getDossierDictionaryVersion();
this.rulesVersion = entityLog.getRulesVersion();
this.legalBasisVersion = entityLog.getLegalBasisVersion();
} **/
}
@NotNull
public static String getDocumentId(String dossierId, String fileId) {
return dossierId + "/" + fileId;
}
}

View File

@ -0,0 +1,112 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode
@FieldDefaults(level = AccessLevel.PRIVATE)
@Document(collection = "entity-log-entries")
public class EntityLogEntryDocument {
@Id
String id;
String entryId;
String entityLogId;
String type;
EntryType entryType;
EntryState state;
String value;
String reason;
String matchedRule;
String legalBasis;
List<Integer> containingNodeId;
String closestHeadline;
String section;
List<Position> positions = new ArrayList<>();
String textBefore;
String textAfter;
int startOffset;
int endOffset;
boolean imageHasTransparency;
boolean dictionaryEntry;
boolean dossierDictionaryEntry;
boolean excluded;
@EqualsAndHashCode.Exclude
List<Change> changes = new ArrayList<>();
@EqualsAndHashCode.Exclude
List<ManualChange> manualChanges = new ArrayList<>();
Set<Engine> engines = new HashSet<>();
Set<String> reference = new HashSet<>();
Set<String> importedRedactionIntersections = new HashSet<>();
int numberOfComments;
public EntityLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry) {
this.id = entityLogId + "/" + entityLogEntry.getId();
this.entryId = entityLogEntry.getId();
this.entityLogId = entityLogId;
this.type = entityLogEntry.getType();
this.entryType = entityLogEntry.getEntryType();
this.state = entityLogEntry.getState();
this.value = entityLogEntry.getValue();
this.reason = entityLogEntry.getReason();
this.matchedRule = entityLogEntry.getMatchedRule();
this.legalBasis = entityLogEntry.getLegalBasis();
this.containingNodeId = new ArrayList<>(entityLogEntry.getContainingNodeId());
this.closestHeadline = entityLogEntry.getClosestHeadline();
this.section = entityLogEntry.getSection();
this.positions = new ArrayList<>(entityLogEntry.getPositions());
this.textBefore = entityLogEntry.getTextBefore();
this.textAfter = entityLogEntry.getTextAfter();
this.startOffset = entityLogEntry.getStartOffset();
this.endOffset = entityLogEntry.getEndOffset();
this.imageHasTransparency = entityLogEntry.isImageHasTransparency();
this.dictionaryEntry = entityLogEntry.isDictionaryEntry();
this.dossierDictionaryEntry = entityLogEntry.isDossierDictionaryEntry();
this.excluded = entityLogEntry.isExcluded();
this.changes = new ArrayList<>(entityLogEntry.getChanges());
this.manualChanges = new ArrayList<>(entityLogEntry.getManualChanges());
this.engines = new HashSet<>(entityLogEntry.getEngines());
this.reference = new HashSet<>(entityLogEntry.getReference());
this.importedRedactionIntersections = new HashSet<>(entityLogEntry.getImportedRedactionIntersections());
this.numberOfComments = entityLogEntry.getNumberOfComments();
}
}

View File

@ -1,9 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.data.mongodb.repository.MongoRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
public interface EntityLogMongoRepository extends MongoRepository<EntityLog, String> {
}

View File

@ -1,29 +1,84 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@Service
public class EntityLogMongoService {
private final EntityLogMongoRepository entityLogMongoRepository;
private final EntityLogDocumentRepository entityLogDocumentRepository;
public EntityLogMongoService(EntityLogMongoRepository entityLogMongoRepository) {this.entityLogMongoRepository = entityLogMongoRepository;}
public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository) {this.entityLogDocumentRepository = entityLogDocumentRepository;}
public EntityLog saveEntityLogDocument(EntityLog entityLogDocument) {
return entityLogMongoRepository.save(entityLogDocument);
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogDocumentRepository.save(new EntityLogDocument(dossierId, fileId, entityLog));
}
public Optional<EntityLog> findEntityLogDocumentById(String id) {
return entityLogMongoRepository.findById(id);
public Optional<EntityLog> findEntityLogByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId)).map(EntityLogMongoService::fromDocument);
}
public boolean entityLogDocumentExists(String id) {
return entityLogMongoRepository.existsById(id);
return entityLogDocumentRepository.existsById(id);
}
private static EntityLog fromDocument(EntityLogDocument entityLogDocument) {
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion());
entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber());
entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument().stream()
.map(EntityLogMongoService::fromDocument)
.collect(Collectors.toList()));
entityLog.setLegalBasis(entityLogDocument.getLegalBasis());
entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion());
entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion());
entityLog.setRulesVersion(entityLogDocument.getRulesVersion());
entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion());
return entityLog;
}
private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) {
EntityLogEntry entityLogEntry = new EntityLogEntry();
entityLogEntry.setId(entityLogEntryDocument.getEntryId());
entityLogEntry.setState(entityLogEntryDocument.getState());
entityLogEntry.setValue(entityLogEntryDocument.getValue());
entityLogEntry.setReason(entityLogEntryDocument.getReason());
entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule());
entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis());
entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId()));
entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline());
entityLogEntry.setSection(entityLogEntryDocument.getSection());
entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions()));
entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore());
entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter());
entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset());
entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset());
entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency());
entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry());
entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry());
entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded());
entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges()));
entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges()));
entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines()));
entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference()));
entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections()));
entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments());
return entityLogEntry;
}
}

View File

@ -1,8 +1,10 @@
package com.iqser.red.service.redaction.v1.server.mongo;
package com.iqser.red.service.redaction.v1.server.mongodeprecated;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;

View File

@ -18,6 +18,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -38,9 +39,6 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
@ -81,7 +79,6 @@ public class AnalyzeService {
ImportedRedactionEntryService importedRedactionEntryService;
ObservedStorageService observedStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
EntityLogDocumentService entityLogDocumentService;
@Timed("redactmanager_reanalyze")
@ -258,8 +255,8 @@ public class AnalyzeService {
Set<FileAttribute> addedFileAttributes) {
EntityLog entityLog = entityLogChanges.getEntityLog();
entityLogDocumentService.saveEntityLogDocument(new EntityLogDocument(analyzeRequest.getFileId(), entityLog));
//redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog());
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.isHasChanges() || !isReanalysis) {

View File

@ -42,7 +42,7 @@ public class EntityChangeLogService {
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getEntryId().equals(entityLogEntry.getEntryId()))
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.findAny();
if (optionalPreviousEntity.isEmpty()) {
hasChanges = true;
@ -66,10 +66,10 @@ public class EntityChangeLogService {
private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber, OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getEntryId)
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !existingIds.contains(entry.getEntryId()))
.filter(entry -> !existingIds.contains(entry.getId()))
.toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));

View File

@ -72,7 +72,7 @@ public class EntityLogCreatorService {
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
return new EntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionServiceSettings.getAnalysisVersion(),
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
@ -118,12 +118,12 @@ public class EntityLogCreatorService {
.get(0)))
.collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getEntryId)
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getEntryId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))))
.collect(Collectors.toList());
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
@ -173,7 +173,7 @@ public class EntityLogCreatorService {
.toList();
// set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
entityLogEntry.setEntryId(positionOnPage.getId());
entityLogEntry.setId(positionOnPage.getId());
entityLogEntry.setPositions(rectanglesPerLine);
entityLogEntries.add(entityLogEntry);
}
@ -186,7 +186,7 @@ public class EntityLogCreatorService {
boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
return EntityLogEntry.builder()
.entryId(image.getId())
.id(image.getId())
.value(image.value())
.type(image.type())
.reason(image.buildReasonWithManualChangeDescriptions())
@ -214,7 +214,7 @@ public class EntityLogCreatorService {
.orElse(precursorEntity.getType());
boolean isHint = isHint(precursorEntity.getEntityType());
return EntityLogEntry.builder()
.entryId(precursorEntity.getId())
.id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())

View File

@ -79,7 +79,7 @@ public class UnprocessedChangesService {
.toList();
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getEntryId()))
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
.toList()
.stream()
.map(PrecursorEntity::fromEntityLogEntry)

View File

@ -53,7 +53,7 @@ public class SectionFinderService {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
Set<Integer> sectionsToReanalyse = new HashSet<>();
for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
if (relevantManuallyModifiedAnnotationIds.contains(entry.getEntryId())) {
if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
if (entry.getContainingNodeId().isEmpty()) {
continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
}

View File

@ -17,7 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
@ -40,7 +40,7 @@ public class RedactionStorageService {
private final StorageService storageService;
private final EntityLogDocumentService entityLogDocumentService;
private final EntityLogMongoService entityLogMongoService;
@SneakyThrows
@ -78,6 +78,13 @@ public class RedactionStorageService {
}
@SneakyThrows
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
}
@Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
@ -136,7 +143,7 @@ public class RedactionStorageService {
try {
//EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
EntityLog entityLog = entityLogDocumentService.findEntityLogDocumentById(fileId).orElseThrow(() -> new StorageObjectDoesNotExist("")).getEntityLog();
EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId).orElseThrow(() -> new StorageObjectDoesNotExist(""));
entityLog.setEntityLogEntry(entityLog.getEntityLogEntry()
.stream()
.filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
@ -205,7 +212,7 @@ public class RedactionStorageService {
public boolean entityLogExists(String dossierId, String fileId) {
//return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG));
return entityLogDocumentService.entityLogDocumentExists(fileId);
return entityLogMongoService.entityLogDocumentExists(fileId);
}

View File

@ -266,7 +266,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
.orElseThrow();
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getEntryId().equals(newId))
.filter(entry -> entry.getId().equals(newId))
.findAny()
.orElseThrow();

View File

@ -41,7 +41,7 @@ import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentRepository;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.mongo._EntityLogDocumentRepository;
import com.iqser.red.service.redaction.v1.server.mongodeprecated._EntityLogDocumentRepository;
import com.iqser.red.storage.commons.service.StorageServiceImpl;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -75,6 +75,7 @@ public class MyOtherMongoTest {
private static final String USERNAME = "root";
private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
private static final String TEST_DOSSIER_ID = "testdossierid";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
public static final String COLLECTION_ENTITY_LOG = "entity-logs";
@ -97,61 +98,6 @@ public class MyOtherMongoTest {
@Test
@SneakyThrows
public void testEntityLogDocumentRepo() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_FILE_ID, entityLog);
entityLogDocumentService.saveEntityLogDocument(entityLogDocument);
//Optional<EntityLogDocument> found = entityLogDocumentService.findEntityLogDocumentById(TEST_FILE_ID);
//assert(found.isPresent());
}
@Test
@SneakyThrows
public void testEntityLogDocumentRepoUsingTemplate() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_FILE_ID, entityLog);
_entityLogDocumentRepository.saveEntityLogDocument(entityLogDocument);
//Optional<EntityLogDocument> found = entityLogDocumentService.findEntityLogDocumentById(TEST_FILE_ID);
//assert(found.isPresent());
}
@Test
@SneakyThrows
public void testEntityLogDocumentRepoUsingTemplateEmpty() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_FILE_ID, new EntityLog());
_entityLogDocumentRepository.saveEntityLogDocument(entityLogDocument);
//Optional<EntityLogDocument> found = entityLogDocumentService.findEntityLogDocumentById(TEST_FILE_ID);
//assert(found.isPresent());
}
@Test
@SneakyThrows
public void testSaveEntityLogDocument() {
@ -162,7 +108,7 @@ public class MyOtherMongoTest {
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.saveEntityLogDocument(entityLog);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
//Optional<EntityLogDocument> found = entityLogDocumentService.findEntityLogDocumentById(TEST_FILE_ID);
//assert(found.isPresent());
@ -173,7 +119,7 @@ public class MyOtherMongoTest {
@SneakyThrows
public void testFindEntityLogDocumentById() {
Optional<EntityLog> entityLogDocumentById = entityLogMongoService.findEntityLogDocumentById(TEST_FILE_ID);
Optional<EntityLog> entityLogDocumentById = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE_ID);
assert(entityLogDocumentById.isPresent());
}
@ -188,7 +134,7 @@ public class MyOtherMongoTest {
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_FILE_ID, entityLog);
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
String jsonData = objectMapper.writeValueAsString(entityLogDocument);
// Convert JSON data to InputStream

View File

@ -141,7 +141,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
var idRemoval = buildIdRemoval(publishedInformationEntry1.getEntryId());
var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
request.setManualRedactions(manualRedactions);
@ -202,7 +202,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
.filter(e -> e.getMatchedRule().startsWith("CBI.16"))
.findAny()
.orElseThrow();
IdRemoval removal = buildIdRemoval(desireeEtAl.getEntryId());
IdRemoval removal = buildIdRemoval(desireeEtAl.getId());
request.setManualRedactions(ManualRedactions.builder().idsToRemove(Set.of(removal)).build());
analyzeService.reanalyze(request);

View File

@ -171,7 +171,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
entityLog.getEntityLogEntry()
.forEach(entry -> {
duplicates.computeIfAbsent(entry.getEntryId(), v -> new ArrayList<>()).add(entry);
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
@ -401,7 +401,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
entityLog.getEntityLogEntry()
.forEach(entry -> {
duplicates.computeIfAbsent(entry.getEntryId(), v -> new ArrayList<>()).add(entry);
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
});
duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1));
@ -1305,7 +1305,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var changedAnnotation = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals("3029651d0842a625f2d23f8375c23600"))
.filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
.findFirst()
.get();
@ -1342,10 +1342,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.APPLIED);
@ -1356,10 +1356,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
@ -1370,18 +1370,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId2)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId2))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.APPLIED);
@ -1395,18 +1395,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId2)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId2))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.REMOVED);
@ -1418,18 +1418,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.APPLIED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId2)));
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId2))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.REMOVED);
}
@ -1455,7 +1455,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
request.setManualRedactions(ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.updateDictionary(true)
.annotationId(david.getEntryId())
.annotationId(david.getId())
.requestDate(OffsetDateTime.now())
.value("David Ksenia")
.positions(List.of(Rectangle.builder()
@ -1472,7 +1472,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var resizedEntity = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getEntryId().equals(david.getEntryId()))
.filter(e -> e.getId().equals(david.getId()))
.findFirst()
.get();
assertEquals(resizedEntity.getState(), EntryState.APPLIED);

View File

@ -464,10 +464,10 @@ public class RulesTest {
for (EntityLogEntry redactionLogEntry : entityLog.getEntityLogEntry()) {
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry()
.stream()
.filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getEntryId()))
.filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId()))
.findFirst();
assertThat(savedRedactionLogEntry).isPresent();
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getEntryId());
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());

View File

@ -152,8 +152,8 @@ public class AnnotationService {
annotation.setRectangle(pdRectangle);
annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles)));
annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry));
annotation.setTitlePopup(redactionLogEntry.getEntryType().name() + ":" + redactionLogEntry.getState().name() + ":" + redactionLogEntry.getEntryId());
annotation.setAnnotationName(redactionLogEntry.getEntryId());
annotation.setTitlePopup(redactionLogEntry.getEntryType().name() + ":" + redactionLogEntry.getState().name() + ":" + redactionLogEntry.getId());
annotation.setAnnotationName(redactionLogEntry.getId());
float[] color;
if (redactionLogEntry.getEntryType().equals(EntryType.RECOMMENDATION)) {
color = new float[]{0, 0.8f, 0};

View File

@ -176,7 +176,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
.get().getEntryId();
.get().getId();
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getResizeRedactions()
.add(ManualResizeRedaction.builder()
@ -203,7 +203,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.filter(entry -> entry.getValue().equals(expandedEntityKeyword))
.findFirst()
.get();
assertEquals(idToResize, resizedEntry.getEntryId());
assertEquals(idToResize, resizedEntry.getId());
assertEquals(1,
redactionLog.getEntityLogEntry()
.stream()
@ -318,7 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
long end = System.currentTimeMillis();
var optionalEntry = redactionLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findAny();
assertTrue(optionalEntry.isPresent());
assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
@ -358,7 +358,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
ManualRecategorization recategorization = ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
.type("vertebrate")
.annotationId(oxfordUniversityPress.getEntryId())
.annotationId(oxfordUniversityPress.getId())
.fileId(TEST_FILE_ID)
.build();
@ -429,7 +429,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getEntryId().equals(annotationId))
.filter(entry -> entry.getId().equals(annotationId))
.findFirst()
.orElseThrow();
assertEquals("Expand to Hint", entityLogEntry.getValue());
@ -458,7 +458,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(dictionaryEntry.getEntryId())
.annotationId(dictionaryEntry.getId())
.requestDate(OffsetDateTime.now())
.value("Image")
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
@ -472,7 +472,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getEntryId().equals(dictionaryEntry.getEntryId()))
.filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
.findFirst()
.orElseThrow();
assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,