diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java index 8fe206d3..89231af1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java @@ -143,6 +143,8 @@ public class SectionTextBuilderService { SearchableText entireTableText = new SearchableText(); SectionText sectionText = new SectionText(); for (List row : table.getRows()) { + int start = 0; + List cellStarts = new ArrayList<>(); for (Cell cell : row) { if (CollectionUtils.isEmpty(cell.getTextBlocks())) { continue; @@ -160,7 +162,13 @@ public class SectionTextBuilderService { for (TextBlock textBlock : cell.getTextBlocks()) { entireTableText.addAll(textBlock.getSequences()); } + + int cellStart = start; + + cellStarts.add(cellStart); + start = start + cell.toString().trim().length() + 1; } + sectionText.setCellStarts(cellStarts); } sectionText.setText(entireTableText.toString()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 6a4a5524..e0409b3c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -664,7 +664,251 @@ public class RedactionIntegrationTest { public void redactionTest() throws IOException { long start = System.currentTimeMillis(); - ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/S11.pdf"); + AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); + request.setExcludedPages(Set.of(1)); + + request.setFileAttributes(List.of(FileAttribute.builder() + .id("fileAttributeId") + .label("Vertebrate Study") + .placeholder("{fileattributes.vertebrateStudy}") + .value("true") + .build())); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + AnalyzeResult result = analyzeService.analyze(request); + + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID); + + redactionLog.getRedactionLogEntry().forEach(entry -> { + if (entry.isImage()) { + System.out.println("---->" + entry.getType()); + } + }); + + long end = System.currentTimeMillis(); + + System.out.println("first analysis duration: " + (end - start)); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) { + fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID))); + } + + int correctFound = 0; + loop: + for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) { + for (SectionText sectionText : text.getSectionTexts()) { + if (redactionLogEntry.isImage()) { + correctFound++; + continue loop; + } + if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) { + String value = sectionText.getText() + .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); + if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { + correctFound++; + } else { + throw new RuntimeException("WTF"); + } + } + } + } + assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size()); + + dictionary.get(AUTHOR).add("properties"); + reanlysisVersions.put("properties", 1L); + + dictionary.get(AUTHOR).add("physical"); + reanlysisVersions.put("physical", 2L); + + deleted.add("David Chubb"); + deleted.add("mouse"); + + dictionary.get(FALSE_POSITIVE).add("David Chubb"); + reanlysisVersions.put("David Chubb", 3L); + + reanlysisVersions.put("mouse", 3L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); + + start = System.currentTimeMillis(); + + ManualRedactions manualRedactions = new ManualRedactions(); + + manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder() + .id("37eee3e9d589a5cc529bfec38c3ba479") + .status(Status.APPROVED) + .type("signature") + .build())); + + request.setManualRedactions(manualRedactions); + + AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + end = System.currentTimeMillis(); + System.out.println("reanalysis analysis duration: " + (end - start)); + + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + + deleted.remove("mouse"); + reanlysisVersions.put("mouse", 4L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + analyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + System.out.println("hi"); + } + + @Test + public void redactionTest1() throws IOException { + + long start = System.currentTimeMillis(); + ClassPathResource pdfFileResource = new ClassPathResource("files/S11.pdf"); + AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); + request.setExcludedPages(Set.of(1)); + + request.setFileAttributes(List.of(FileAttribute.builder() + .id("fileAttributeId") + .label("Vertebrate Study") + .placeholder("{fileattributes.vertebrateStudy}") + .value("true") + .build())); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + AnalyzeResult result = analyzeService.analyze(request); + + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID); + + redactionLog.getRedactionLogEntry().forEach(entry -> { + if (entry.isImage()) { + System.out.println("---->" + entry.getType()); + } + }); + + long end = System.currentTimeMillis(); + + System.out.println("first analysis duration: " + (end - start)); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) { + fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID))); + } + + int correctFound = 0; + loop: + for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) { + for (SectionText sectionText : text.getSectionTexts()) { + if (redactionLogEntry.isImage()) { + correctFound++; + continue loop; + } + if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) { + String value = sectionText.getText() + .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); + if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { + correctFound++; + } else { + throw new RuntimeException("WTF"); + } + } + } + } + assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size()); + + dictionary.get(AUTHOR).add("properties"); + reanlysisVersions.put("properties", 1L); + + dictionary.get(AUTHOR).add("physical"); + reanlysisVersions.put("physical", 2L); + + deleted.add("David Chubb"); + deleted.add("mouse"); + + dictionary.get(FALSE_POSITIVE).add("David Chubb"); + reanlysisVersions.put("David Chubb", 3L); + + reanlysisVersions.put("mouse", 3L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); + + start = System.currentTimeMillis(); + + ManualRedactions manualRedactions = new ManualRedactions(); + + manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder() + .id("37eee3e9d589a5cc529bfec38c3ba479") + .status(Status.APPROVED) + .type("signature") + .build())); + + request.setManualRedactions(manualRedactions); + + AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + end = System.currentTimeMillis(); + System.out.println("reanalysis analysis duration: " + (end - start)); + + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + + deleted.remove("mouse"); + reanlysisVersions.put("mouse", 4L); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L); + + when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)) + .thenReturn(getDictionaryResponse(VERTEBRATE, false)); + + analyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + System.out.println("hi"); + } + + @Test + public void redactionTest2() throws IOException { + + long start = System.currentTimeMillis(); + ClassPathResource pdfFileResource = new ClassPathResource("files/S16.pdf"); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setExcludedPages(Set.of(1)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/S11.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/S11.pdf new file mode 100644 index 00000000..42d37883 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/S11.pdf differ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/S16.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/S16.pdf new file mode 100644 index 00000000..013951cc Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/S16.pdf differ