Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702

Browse Source
This commit is contained in:
maverickstuder 2024-03-13 16:07:22 +01:00
parent 1a11d3b509
commit 85d456d941
18 changed files with 155 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -229,7 +229,7 @@ public final class MigrationEntity {
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
return EntityLogEntry.builder()
.id(image.getId())
.entryId(image.getId())
.value(image.value())
.type(image.type())
.reason(image.buildReasonWithManualChangeDescriptions())
@ -254,7 +254,7 @@ public final class MigrationEntity {
public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
return EntityLogEntry.builder()
.id(precursorEntity.getId())
.entryId(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())
@ -287,7 +287,7 @@ public final class MigrationEntity {
assert entity.getPositionsOnPagePerPage().size() == 1;
List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
return EntityLogEntry.builder()
.id(entity.getId())
.entryId(entity.getId())
.positions(rectanglesPerLine)
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java
View File

@ -92,7 +92,7 @@ public class PrecursorEntity implements IEntity {
.toList();
EntityType entityType = getEntityType(entityLogEntry.getEntryType());
return PrecursorEntity.builder()
.id(entityLogEntry.getId())
.id(entityLogEntry.getEntryId())
.value(entityLogEntry.getValue())
.entityPosition(rectangleWithPages)
.ruleIdentifier(entityLogEntry.getMatchedRule())

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java
View File

@ -71,7 +71,7 @@ public class Entity {
public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
return Entity.builder()
.id(e.getId())
.id(e.getEntryId())
.type(e.getType())
.entryType(e.getEntryType())
.state(e.getState())

58
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/CascadeCallback.java Normal file
View File

@ -0,0 +1,58 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.lang.reflect.Field;
import java.lang.reflect.InaccessibleObjectException;
import java.util.Collection;
import org.bouncycastle.util.Iterable;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.util.ReflectionUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.CascadeSave;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
public class CascadeCallback implements ReflectionUtils.FieldCallback {
private Object source;
private MongoTemplate mongoTemplate;
@Override
public void doWith(@NotNull final Field field) throws IllegalArgumentException, IllegalAccessException {
try {
ReflectionUtils.makeAccessible(field);
if (field.isAnnotationPresent(DBRef.class) && field.isAnnotationPresent(CascadeSave.class)) {
final Object fieldValue = field.get(getSource());
if(Collection.class.isAssignableFrom(field.getType())) {
Collection<?> collection = (Collection<?>) fieldValue;
mongoTemplate.insertAll(collection);
}
else {
if (fieldValue != null) {
//final FieldCallback callback = new FieldCallback();
//ReflectionUtils.doWithFields(fieldValue.getClass(), callback);
mongoTemplate.insert(fieldValue);
}
}
}
} catch (InaccessibleObjectException ignored) {
}
}
}

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/CascadeSaveMongoEventListener.java Normal file
View File

@ -0,0 +1,21 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener;
import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent;
import org.springframework.util.ReflectionUtils;
public class CascadeSaveMongoEventListener extends AbstractMongoEventListener<Object> {
@Autowired
private MongoTemplate mongoTemplate;
@Override
public void onBeforeConvert(BeforeConvertEvent<Object> event) {
Object source = event.getSource();
ReflectionUtils.doWithFields(source.getClass(),
new CascadeCallback(source, mongoTemplate));
}
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocumentService.java
View File

@ -31,4 +31,5 @@ public class EntityLogDocumentService {
public boolean entityLogDocumentExists(String id) {
return entityLogDocumentRepository.existsById(id);
}
}

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoRepository.java Normal file
View File

@ -0,0 +1,9 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import org.springframework.data.mongodb.repository.MongoRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
public interface EntityLogMongoRepository extends MongoRepository<EntityLog, String> {
}

35
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/FieldCallback.java Normal file
View File

@ -0,0 +1,35 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.lang.reflect.Field;
import java.lang.reflect.InaccessibleObjectException;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.annotation.Id;
import org.springframework.util.ReflectionUtils;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FieldCallback implements ReflectionUtils.FieldCallback {
private boolean idFound;
@Override
public void doWith(@NotNull final Field field) throws IllegalArgumentException {
try {
ReflectionUtils.makeAccessible(field);
if (field.isAnnotationPresent(Id.class)) {
idFound = true;
}
} catch (InaccessibleObjectException ignored) {
}
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -18,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -39,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocument;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -42,7 +42,7 @@ public class EntityChangeLogService {
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.filter(entry -> entry.getEntryId().equals(entityLogEntry.getEntryId()))
.findAny();
if (optionalPreviousEntity.isEmpty()) {
hasChanges = true;
@ -66,10 +66,10 @@ public class EntityChangeLogService {
private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber, OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.map(EntityLogEntry::getEntryId)
.collect(Collectors.toSet());
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !existingIds.contains(entry.getId()))
.filter(entry -> !existingIds.contains(entry.getEntryId()))
.toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -72,7 +72,7 @@ public class EntityLogCreatorService {
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
return new EntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(),
entityLogEntries,
toEntityLogLegalBasis(legalBasis),
@ -118,12 +118,12 @@ public class EntityLogCreatorService {
.get(0)))
.collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.map(EntityLogEntry::getEntryId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.filter(entry -> (newEntityIds.contains(entry.getEntryId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))))
.collect(Collectors.toList());
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
@ -173,7 +173,7 @@ public class EntityLogCreatorService {
.toList();
// set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
entityLogEntry.setId(positionOnPage.getId());
entityLogEntry.setEntryId(positionOnPage.getId());
entityLogEntry.setPositions(rectanglesPerLine);
entityLogEntries.add(entityLogEntry);
}
@ -187,7 +187,7 @@ public class EntityLogCreatorService {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return EntityLogEntry.builder()
.id(image.getId())
.entryId(image.getId())
.value(image.value())
.type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions())
@ -215,7 +215,7 @@ public class EntityLogCreatorService {
.orElse(precursorEntity.getType());
boolean isHint = isHint(precursorEntity.getEntityType());
return EntityLogEntry.builder()
.id(precursorEntity.getId())
.entryId(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.value(precursorEntity.value())

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java
View File

@ -79,7 +79,7 @@ public class UnprocessedChangesService {
.toList();
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getEntryId()))
.toList()
.stream()
.map(PrecursorEntity::fromEntityLogEntry)

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -53,7 +53,7 @@ public class SectionFinderService {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
Set<Integer> sectionsToReanalyse = new HashSet<>();
for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
if (relevantManuallyModifiedAnnotationIds.contains(entry.getEntryId())) {
if (entry.getContainingNodeId().isEmpty()) {
continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java
View File

@ -266,7 +266,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
.orElseThrow();
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(newId))
.filter(entry -> entry.getEntryId().equals(newId))
.findAny()
.orElseThrow();

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -143,7 +143,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
var idRemoval = buildIdRemoval(publishedInformationEntry1.getEntryId());
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
request.setManualRedactions(manualRedactions);
@ -206,7 +206,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
.filter(e -> e.getMatchedRule().startsWith("CBI.16"))
.findAny()
.orElseThrow();
IdRemoval removal = buildIdRemoval(desireeEtAl.getId());
IdRemoval removal = buildIdRemoval(desireeEtAl.getEntryId());
request.setManualRedactions(ManualRedactions.builder().idsToRemove(Set.of(removal)).build());
analyzeService.reanalyze(request);

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -464,10 +464,10 @@ public class RulesTest {
for (EntityLogEntry redactionLogEntry : entityLog.getEntityLogEntry()) {
var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry()
.stream()
.filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId()))
.filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getEntryId()))
.findFirst();
assertThat(savedRedactionLogEntry).isPresent();
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getEntryId());
assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java
View File

@ -152,8 +152,8 @@ public class AnnotationService {
annotation.setRectangle(pdRectangle);
annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles)));
annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry));
annotation.setTitlePopup(redactionLogEntry.getEntryType().name() + ":" + redactionLogEntry.getState().name() + ":" + redactionLogEntry.getId());
annotation.setAnnotationName(redactionLogEntry.getId());
annotation.setTitlePopup(redactionLogEntry.getEntryType().name() + ":" + redactionLogEntry.getState().name() + ":" + redactionLogEntry.getEntryId());
annotation.setAnnotationName(redactionLogEntry.getEntryId());
float[] color;
if (redactionLogEntry.getEntryType().equals(EntryType.RECOMMENDATION)) {
color = new float[]{0, 0.8f, 0};

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -176,7 +176,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.stream()
.filter(entry -> entry.getValue().equals(testEntityValue1))
.max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
.get().getId();
.get().getEntryId();
ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.getResizeRedactions()
.add(ManualResizeRedaction.builder()
@ -204,7 +204,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.filter(entry -> entry.getValue().equals(expandedEntityKeyword))
.findFirst()
.get();
assertEquals(idToResize, resizedEntry.getId());
assertEquals(idToResize, resizedEntry.getEntryId());
assertEquals(1,
redactionLog.getEntityLogEntry()
.stream()
@ -319,7 +319,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
long end = System.currentTimeMillis();
var optionalEntry = redactionLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.filter(entityLogEntry -> entityLogEntry.getEntryId().equals(manualAddId))
.findAny();
assertTrue(optionalEntry.isPresent());
assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
@ -359,7 +359,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
ManualRecategorization recategorization = ManualRecategorization.builder()
.requestDate(OffsetDateTime.now())
.type("vertebrate")
.annotationId(oxfordUniversityPress.getId())
.annotationId(oxfordUniversityPress.getEntryId())
.fileId(TEST_FILE_ID)
.build();
@ -430,7 +430,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
.filter(entry -> entry.getEntryId().equals(annotationId))
.findFirst()
.orElseThrow();
assertEquals("Expand to Hint", entityLogEntry.getValue());
@ -459,7 +459,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
.annotationId(dictionaryEntry.getId())
.annotationId(dictionaryEntry.getEntryId())
.requestDate(OffsetDateTime.now())
.value("Image")
.positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
@ -473,7 +473,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
.filter(entry -> entry.getEntryId().equals(dictionaryEntry.getEntryId()))
.findFirst()
.orElseThrow();
assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,