Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8278: Fixed skipped signatures after manual changes

Browse Source
This commit is contained in:
Dominique Eifländer 2024-01-18 13:27:14 +01:00
parent 91766f5b0f
commit 9dbd2ac260
4 changed files with 904 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -1,7 +1,5 @@
package com.iqser.red.service.redaction.v1.server.service;
import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@ -120,7 +118,9 @@ public class EntityLogCreatorService {
Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries).stream()
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))
.collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()

56
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1185,6 +1185,44 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
}
@Test
public void signaturesAreRedactionAfterReanalyse() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
analyzeService.analyze(request);
System.out.println("Finished analysis");
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.build());
analyzeService.reanalyze(request);
System.out.println("Finished reanalysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> {
assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
});
}
@Test
@SneakyThrows
public void testRemovingAndAddingRedactionOnTheSameValue() {
@ -1194,8 +1232,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
String manualAddId = UUID.randomUUID().toString();
String manualAddId2 = UUID.randomUUID().toString();
List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build());
ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId, positions, "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2, positions, "commercial information revealing sourcing, market shares or business strategy of the applicant");
ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId,
positions,
"the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2,
positions,
"commercial information revealing sourcing, market shares or business strategy of the applicant");
IdRemoval idRemoval = getIdRemoval(manualAddId);
IdRemoval idRemoval2 = getIdRemoval(manualAddId2);
@ -1228,7 +1270,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build());
request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
.idsToRemove(Set.of(idRemoval, idRemoval2))
.build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
@ -1239,7 +1284,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build());
request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
.idsToRemove(Set.of(idRemoval, idRemoval2))
.build());
analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);

849
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json Normal file
View File

@ -0,0 +1,849 @@
{
"dossierId": "94ce568c-e788-4c1d-ac6b-c1a139e953b3",
"fileId": "f780bb084ac1c5ad2110db68df6a0ce7",
"targetFileExtension": "ORIGIN.pdf.gz",
"responseFileExtension": "IMAGE_INFO.json.gz",
"X-TENANT-ID": "redaction",
"data": [
{
"classification": {
"label": "signature",
"probabilities": {
"signature": 1.0,
"formula": 0.0,
"logo": 0.0,
"other": 0.0
}
},
"representation": "FFFF4F3E1800E1C77FFDF7FFD",
"position": {
"x1": 61,
"x2": 216,
"y1": 523,
"y2": 583,
"pageNumber": 14
},
"geometry": {
"width": 155,
"height": 60
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.1385,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 2.5833,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "signature",
"probabilities": {
"signature": 1.0,
"formula": 0.0,
"logo": 0.0,
"other": 0.0
}
},
"representation": "FFFEF144801140802FFFFFFFF",
"position": {
"x1": 64,
"x2": 203,
"y1": 619,
"y2": 680,
"pageNumber": 14
},
"geometry": {
"width": 139,
"height": 61
},
"alpha": true,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.1323,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 2.2787,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "logo",
"probabilities": {
"logo": 1.0,
"formula": 0.0,
"other": 0.0,
"signature": 0.0
}
},
"representation": "FF300FDF3F78F1E78F1EFCFFF",
"position": {
"x1": 71,
"x2": 268,
"y1": 277,
"y2": 419,
"pageNumber": 14
},
"geometry": {
"width": 197,
"height": 142
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2402,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.3873,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "logo",
"probabilities": {
"logo": 0.9624,
"other": 0.0221,
"formula": 0.0116,
"signature": 0.0039
}
},
"representation": "00CFE000F1CC033CC033CF0E1",
"position": {
"x1": 75,
"x2": 254,
"y1": 117,
"y2": 249,
"pageNumber": 14
},
"geometry": {
"width": 179,
"height": 132
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2208,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.3561,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "logo",
"probabilities": {
"logo": 1.0,
"formula": 0.0,
"other": 0.0,
"signature": 0.0
}
},
"representation": "FFFFFFF20C00FBFFFFFBFFFFF",
"position": {
"x1": 284,
"x2": 562,
"y1": 140,
"y2": 223,
"pageNumber": 14
},
"geometry": {
"width": 278,
"height": 83
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2182,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 3.3494,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "signature",
"probabilities": {
"signature": 1.0,
"formula": 0.0,
"logo": 0.0,
"other": 0.0
}
},
"representation": "FFFFDF7FFFC060C1A7CE3BFFF",
"position": {
"x1": 287,
"x2": 486,
"y1": 613,
"y2": 681,
"pageNumber": 14
},
"geometry": {
"width": 199,
"height": 68
},
"alpha": true,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.1671,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 2.9265,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "signature",
"probabilities": {
"signature": 0.9984,
"logo": 0.0016,
"formula": 0.0,
"other": 0.0
}
},
"representation": "FFFFBFFEF81078E1874FCFBFF",
"position": {
"x1": 296,
"x2": 462,
"y1": 499,
"y2": 594,
"pageNumber": 14
},
"geometry": {
"width": 166,
"height": 95
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.1804,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.7474,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "logo",
"probabilities": {
"logo": 1.0,
"formula": 0.0,
"other": 0.0,
"signature": 0.0
}
},
"representation": "10C00FFF0C30F0C33FCCB5FFF",
"position": {
"x1": 358,
"x2": 538,
"y1": 274,
"y2": 401,
"pageNumber": 14
},
"geometry": {
"width": 180,
"height": 127
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2172,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.4173,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "formula",
"probabilities": {
"formula": 1.0,
"logo": 0.0,
"other": 0.0,
"signature": 0.0
}
},
"representation": "FFF3FFDF0C3AF0C35E4FFFFFF",
"position": {
"x1": 58,
"x2": 212,
"y1": 295,
"y2": 448,
"pageNumber": 15
},
"geometry": {
"width": 154,
"height": 153
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2205,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.0065,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "formula",
"probabilities": {
"formula": 0.3855,
"other": 0.3396,
"logo": 0.1674,
"signature": 0.1074
}
},
"representation": "FA30F080F0C031CE17E8CF237",
"position": {
"x1": 61,
"x2": 210,
"y1": 538,
"y2": 687,
"pageNumber": 15
},
"geometry": {
"width": 149,
"height": 149
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.214,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.0,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": true
},
"allPassed": false
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 0.958,
"formula": 0.0272,
"signature": 0.0137,
"logo": 0.001
}
},
"representation": "FFCF08C04100000E0CF7FFF3F",
"position": {
"x1": 219,
"x2": 396,
"y1": 537,
"y2": 687,
"pageNumber": 15
},
"geometry": {
"width": 177,
"height": 150
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.234,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.18,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "formula",
"probabilities": {
"formula": 1.0,
"logo": 0.0,
"other": 0.0,
"signature": 0.0
}
},
"representation": "7EFBD3064FBD7084604DF4E33",
"position": {
"x1": 240,
"x2": 384,
"y1": 298,
"y2": 413,
"pageNumber": 15
},
"geometry": {
"width": 144,
"height": 115
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.1848,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.2522,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "formula",
"probabilities": {
"formula": 0.7103,
"other": 0.1487,
"logo": 0.1336,
"signature": 0.0074
}
},
"representation": "FEF9CFAFBE10400DA3BE42FBE",
"position": {
"x1": 399,
"x2": 579,
"y1": 300,
"y2": 422,
"pageNumber": 15
},
"geometry": {
"width": 180,
"height": 122
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2129,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.4754,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 0.4883,
"signature": 0.2465,
"formula": 0.1399,
"logo": 0.1253
}
},
"representation": "02080770F3CC810403FDC9B7C",
"position": {
"x1": 406,
"x2": 576,
"y1": 522,
"y2": 686,
"pageNumber": 15
},
"geometry": {
"width": 170,
"height": 164
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2398,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.0366,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": true
},
"allPassed": false
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 1.0,
"formula": 0.0,
"logo": 0.0,
"signature": 0.0
}
},
"representation": "400C3E39776E9A706954EF1",
"position": {
"x1": 56,
"x2": 252,
"y1": 534,
"y2": 658,
"pageNumber": 16
},
"geometry": {
"width": 196,
"height": 124
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2239,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.5806,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 0.9799,
"formula": 0.0162,
"signature": 0.0039,
"logo": 0.0
}
},
"representation": "83CE891C0C78B1E4831EBCF3F",
"position": {
"x1": 60,
"x2": 300,
"y1": 105,
"y2": 325,
"pageNumber": 16
},
"geometry": {
"width": 240,
"height": 220
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.33,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.0909,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 0.9961,
"logo": 0.002,
"formula": 0.001,
"signature": 0.0009
}
},
"representation": "FF3000040031C02FBFFEFFF0C",
"position": {
"x1": 126,
"x2": 326,
"y1": 397,
"y2": 509,
"pageNumber": 16
},
"geometry": {
"width": 200,
"height": 112
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.215,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.7857,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 1.0,
"formula": 0.0,
"logo": 0.0,
"signature": 0.0
}
},
"representation": "8C14742030E4CF1F6EBBFFDC3",
"position": {
"x1": 313,
"x2": 549,
"y1": 144,
"y2": 302,
"pageNumber": 16
},
"geometry": {
"width": 236,
"height": 158
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2774,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.4937,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 1.0,
"formula": 0.0,
"logo": 0.0,
"signature": 0.0
}
},
"representation": "F7404EB6010C067090E0FFDF7",
"position": {
"x1": 342,
"x2": 509,
"y1": 524,
"y2": 662,
"pageNumber": 16
},
"geometry": {
"width": 167,
"height": 138
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.2181,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.2101,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
},
{
"classification": {
"label": "other",
"probabilities": {
"other": 0.9704,
"formula": 0.0269,
"signature": 0.0021,
"logo": 0.0007
}
},
"representation": "1F72F14C033C30F0F3AF0FFCF",
"position": {
"x1": 376,
"x2": 546,
"y1": 577,
"y2": 679,
"pageNumber": 19
},
"geometry": {
"width": 170,
"height": 102
},
"alpha": false,
"filters": {
"geometry": {
"imageSize": {
"quotient": 0.1891,
"tooLarge": false,
"tooSmall": false
},
"imageFormat": {
"quotient": 1.6667,
"tooTall": false,
"tooWide": false
}
},
"probability": {
"unconfident": false
},
"allPassed": true
}
}
]
}

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf Normal file
View File

Binary file not shown.