Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3a0ffaef72 | ||
|
|
bb3d6f5d74 |
@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
|
|||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
implementation("org.springframework:spring-web:6.0.12")
|
implementation("org.springframework:spring-web:6.0.12")
|
||||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.338.0")
|
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
|
||||||
}
|
}
|
||||||
|
|
||||||
publishing {
|
publishing {
|
||||||
|
|||||||
@ -16,7 +16,7 @@ val layoutParserVersion = "0.86.0"
|
|||||||
val jacksonVersion = "2.15.2"
|
val jacksonVersion = "2.15.2"
|
||||||
val droolsVersion = "9.44.0.Final"
|
val droolsVersion = "9.44.0.Final"
|
||||||
val pdfBoxVersion = "3.0.0"
|
val pdfBoxVersion = "3.0.0"
|
||||||
val persistenceServiceVersion = "2.338.0"
|
val persistenceServiceVersion = "2.351.0"
|
||||||
val springBootStarterVersion = "3.1.5"
|
val springBootStarterVersion = "3.1.5"
|
||||||
|
|
||||||
configurations {
|
configurations {
|
||||||
|
|||||||
@ -80,7 +80,8 @@ public class ManualChangeOverwrite {
|
|||||||
manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
|
manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
|
||||||
updateFields(manualChanges);
|
updateFields(manualChanges);
|
||||||
// make list unmodifiable.
|
// make list unmodifiable.
|
||||||
return manualChanges.stream().toList();
|
return manualChanges.stream()
|
||||||
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -121,14 +122,18 @@ public class ManualChangeOverwrite {
|
|||||||
resized = true;
|
resized = true;
|
||||||
// This is only for not found Manual Entities.
|
// This is only for not found Manual Entities.
|
||||||
value = manualResizeRedaction.getValue();
|
value = manualResizeRedaction.getValue();
|
||||||
positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
|
positions = manualResizeRedaction.getPositions()
|
||||||
|
.stream()
|
||||||
|
.map(RectangleWithPage::fromAnnotationRectangle)
|
||||||
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (manualChange instanceof ManualRecategorization recategorization) {
|
if (manualChange instanceof ManualRecategorization recategorization) {
|
||||||
// recategorization logic happens in ManualChangesApplicationService.
|
|
||||||
recategorized = true;
|
recategorized = true;
|
||||||
// this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
|
|
||||||
type = recategorization.getType();
|
type = recategorization.getType();
|
||||||
|
if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
|
||||||
|
legalBasis = recategorization.getLegalBasis();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));
|
descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));
|
||||||
|
|||||||
@ -4,8 +4,8 @@ import java.time.OffsetDateTime;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||||
@ -13,8 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
|
|
||||||
|
|
||||||
import lombok.experimental.UtilityClass;
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
@ -23,7 +21,9 @@ public class ManualChangeFactory {
|
|||||||
|
|
||||||
public List<ManualChange> toManualChangeList(List<BaseAnnotation> manualChanges, boolean isHint) {
|
public List<ManualChange> toManualChangeList(List<BaseAnnotation> manualChanges, boolean isHint) {
|
||||||
|
|
||||||
return manualChanges.stream().map(baseAnnotation -> toManualChange(baseAnnotation, isHint)).collect(Collectors.toList());
|
return manualChanges.stream()
|
||||||
|
.map(baseAnnotation -> toManualChange(baseAnnotation, isHint))
|
||||||
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -37,7 +37,8 @@ public class ManualChangeFactory {
|
|||||||
} else if (baseAnnotation instanceof ManualForceRedaction) {
|
} else if (baseAnnotation instanceof ManualForceRedaction) {
|
||||||
manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
|
manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
|
||||||
} else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
|
} else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
|
||||||
manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue());
|
manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
|
||||||
|
.withChange("value", manualResizeRedact.getValue());
|
||||||
} else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
|
} else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
|
||||||
manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
|
manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
|
||||||
.withChange("value", manualRedactionEntry.getValue());
|
.withChange("value", manualRedactionEntry.getValue());
|
||||||
|
|||||||
@ -57,16 +57,21 @@ public class SectionFinderService {
|
|||||||
if (entry.getContainingNodeId().isEmpty()) {
|
if (entry.getContainingNodeId().isEmpty()) {
|
||||||
continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
|
continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
|
||||||
}
|
}
|
||||||
sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
|
sectionsToReanalyse.add(entry.getContainingNodeId()
|
||||||
|
.get(0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
|
var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
|
||||||
true);
|
.stream()
|
||||||
|
.map(DictionaryIncrementValue::getValue)
|
||||||
|
.toList(), true);
|
||||||
|
|
||||||
document.streamChildren().forEach(mainNode -> {
|
document.streamChildren()
|
||||||
|
.forEach(mainNode -> {
|
||||||
if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
|
if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
|
||||||
sectionsToReanalyse.add(mainNode.getTreeId().get(0));
|
sectionsToReanalyse.add(mainNode.getTreeId()
|
||||||
|
.get(0));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -87,8 +92,11 @@ public class SectionFinderService {
|
|||||||
return document.getPages()
|
return document.getPages()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
|
.filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
|
||||||
.flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
|
.flatMap(page -> Stream.concat(page.getMainBody()
|
||||||
.map(node -> node.getTreeId().get(0))
|
.stream()
|
||||||
|
.filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
|
||||||
|
.map(node -> node.getTreeId()
|
||||||
|
.get(0))
|
||||||
.toList();
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -99,8 +107,7 @@ public class SectionFinderService {
|
|||||||
return Collections.emptySet();
|
return Collections.emptySet();
|
||||||
}
|
}
|
||||||
|
|
||||||
return analyzeRequest.getManualRedactions()
|
return analyzeRequest.getManualRedactions().getEntriesToAdd()
|
||||||
.getEntriesToAdd()
|
|
||||||
.stream()
|
.stream()
|
||||||
.filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
|
.filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
|
||||||
.map(ManualRedactionEntry::getPositions)
|
.map(ManualRedactionEntry::getPositions)
|
||||||
@ -132,11 +139,22 @@ public class SectionFinderService {
|
|||||||
return new HashSet<>();
|
return new HashSet<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
|
return Stream.concat(manualRedactions.getResizeRedactions()
|
||||||
Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
|
.stream()
|
||||||
Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
|
.map(ManualResizeRedaction::getAnnotationId),
|
||||||
Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
|
Stream.concat(manualRedactions.getLegalBasisChanges()
|
||||||
manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
|
.stream()
|
||||||
|
.map(ManualLegalBasisChange::getAnnotationId),
|
||||||
|
Stream.concat(manualRedactions.getRecategorizations()
|
||||||
|
.stream()
|
||||||
|
.map(ManualRecategorization::getAnnotationId),
|
||||||
|
Stream.concat(manualRedactions.getIdsToRemove()
|
||||||
|
.stream()
|
||||||
|
.map(IdRemoval::getAnnotationId),
|
||||||
|
manualRedactions.getForceRedactions()
|
||||||
|
.stream()
|
||||||
|
.map(ManualForceRedaction::getAnnotationId)))))
|
||||||
|
.collect(Collectors.toSet());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1236,6 +1236,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
.recategorizations(Set.of(ManualRecategorization.builder()
|
.recategorizations(Set.of(ManualRecategorization.builder()
|
||||||
.annotationId("3029651d0842a625f2d23f8375c23600")
|
.annotationId("3029651d0842a625f2d23f8375c23600")
|
||||||
.type("CBI_author")
|
.type("CBI_author")
|
||||||
|
.legalBasis("")
|
||||||
.requestDate(OffsetDateTime.now())
|
.requestDate(OffsetDateTime.now())
|
||||||
.fileId(TEST_FILE_ID)
|
.fileId(TEST_FILE_ID)
|
||||||
.build()))
|
.build()))
|
||||||
@ -1331,6 +1332,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
|
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testResizeWithUpdateDictionaryTrue() {
|
public void testResizeWithUpdateDictionaryTrue() {
|
||||||
@ -1364,12 +1366,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
private IdRemoval getIdRemoval(String id) {
|
private IdRemoval getIdRemoval(String id) {
|
||||||
|
|
||||||
return IdRemoval.builder()
|
return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
|
||||||
.annotationId(id)
|
|
||||||
.removeFromAllDossiers(false)
|
|
||||||
.removeFromDictionary(false)
|
|
||||||
.requestDate(OffsetDateTime.now())
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -115,7 +115,12 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
|
|||||||
assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
|
assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
|
||||||
assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()));
|
assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()));
|
||||||
|
|
||||||
ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder().type("type").requestDate(start.plusSeconds(5)).annotationId(annotationId).build();
|
ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder()
|
||||||
|
.type("type")
|
||||||
|
.requestDate(start.plusSeconds(5))
|
||||||
|
.annotationId(annotationId)
|
||||||
|
.legalBasis("")
|
||||||
|
.build();
|
||||||
entity.getManualOverwrite().addChange(imageRecategorizationRequest);
|
entity.getManualOverwrite().addChange(imageRecategorizationRequest);
|
||||||
assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
|
assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
|
||||||
assertTrue(entity.getManualOverwrite().getRecategorized().get());
|
assertTrue(entity.getManualOverwrite().getRecategorized().get());
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user