RED-2167: added computation of cellstarts for tables with two colums

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -143,6 +143,8 @@ public class SectionTextBuilderService {
         SearchableText entireTableText = new SearchableText();
         SectionText sectionText = new SectionText();
         for (List<Cell> row : table.getRows()) {
+            int start = 0;
+            List<Integer> cellStarts = new ArrayList<>();
             for (Cell cell : row) {
                 if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
                     continue;
@@ -160,7 +162,13 @@ public class SectionTextBuilderService {
                 for (TextBlock textBlock : cell.getTextBlocks()) {
                     entireTableText.addAll(textBlock.getSequences());
                 }
+
+                int cellStart = start;
+
+                cellStarts.add(cellStart);
+                start = start + cell.toString().trim().length() + 1;
             }
+            sectionText.setCellStarts(cellStarts);
         }
 
         sectionText.setText(entireTableText.toString());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -664,7 +664,251 @@ public class RedactionIntegrationTest {
     public void redactionTest() throws IOException {
 
         long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/S11.pdf");
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+        request.setExcludedPages(Set.of(1));
+
+        request.setFileAttributes(List.of(FileAttribute.builder()
+                .id("fileAttributeId")
+                .label("Vertebrate Study")
+                .placeholder("{fileattributes.vertebrateStudy}")
+                .value("true")
+                .build()));
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        redactionLog.getRedactionLogEntry().forEach(entry -> {
+            if (entry.isImage()) {
+                System.out.println("---->" + entry.getType());
+            }
+        });
+
+        long end = System.currentTimeMillis();
+
+        System.out.println("first analysis duration: " + (end - start));
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
+            fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
+        }
+
+        int correctFound = 0;
+        loop:
+        for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
+            for (SectionText sectionText : text.getSectionTexts()) {
+                if (redactionLogEntry.isImage()) {
+                    correctFound++;
+                    continue loop;
+                }
+                if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
+                    String value = sectionText.getText()
+                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
+                    if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
+                        correctFound++;
+                    } else {
+                        throw new RuntimeException("WTF");
+                    }
+                }
+            }
+        }
+        assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
+
+        dictionary.get(AUTHOR).add("properties");
+        reanlysisVersions.put("properties", 1L);
+
+        dictionary.get(AUTHOR).add("physical");
+        reanlysisVersions.put("physical", 2L);
+
+        deleted.add("David Chubb");
+        deleted.add("mouse");
+
+        dictionary.get(FALSE_POSITIVE).add("David Chubb");
+        reanlysisVersions.put("David Chubb", 3L);
+
+        reanlysisVersions.put("mouse", 3L);
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
+                .thenReturn(getDictionaryResponse(VERTEBRATE, false));
+
+        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
+                .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
+
+        start = System.currentTimeMillis();
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder()
+                .id("37eee3e9d589a5cc529bfec38c3ba479")
+                .status(Status.APPROVED)
+                .type("signature")
+                .build()));
+
+        request.setManualRedactions(manualRedactions);
+
+        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+
+        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        end = System.currentTimeMillis();
+        System.out.println("reanalysis analysis duration: " + (end - start));
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+
+        deleted.remove("mouse");
+        reanlysisVersions.put("mouse", 4L);
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
+                .thenReturn(getDictionaryResponse(VERTEBRATE, false));
+
+        analyzeService.reanalyze(request);
+
+        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        System.out.println("hi");
+    }
+
+    @Test
+    public void redactionTest1() throws IOException {
+
+        long start = System.currentTimeMillis();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/S11.pdf");
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+        request.setExcludedPages(Set.of(1));
+
+        request.setFileAttributes(List.of(FileAttribute.builder()
+                .id("fileAttributeId")
+                .label("Vertebrate Study")
+                .placeholder("{fileattributes.vertebrateStudy}")
+                .value("true")
+                .build()));
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        redactionLog.getRedactionLogEntry().forEach(entry -> {
+            if (entry.isImage()) {
+                System.out.println("---->" + entry.getType());
+            }
+        });
+
+        long end = System.currentTimeMillis();
+
+        System.out.println("first analysis duration: " + (end - start));
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) {
+            fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
+        }
+
+        int correctFound = 0;
+        loop:
+        for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
+            for (SectionText sectionText : text.getSectionTexts()) {
+                if (redactionLogEntry.isImage()) {
+                    correctFound++;
+                    continue loop;
+                }
+                if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
+                    String value = sectionText.getText()
+                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
+                    if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
+                        correctFound++;
+                    } else {
+                        throw new RuntimeException("WTF");
+                    }
+                }
+            }
+        }
+        assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
+
+        dictionary.get(AUTHOR).add("properties");
+        reanlysisVersions.put("properties", 1L);
+
+        dictionary.get(AUTHOR).add("physical");
+        reanlysisVersions.put("physical", 2L);
+
+        deleted.add("David Chubb");
+        deleted.add("mouse");
+
+        dictionary.get(FALSE_POSITIVE).add("David Chubb");
+        reanlysisVersions.put("David Chubb", 3L);
+
+        reanlysisVersions.put("mouse", 3L);
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(3L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
+                .thenReturn(getDictionaryResponse(VERTEBRATE, false));
+
+        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
+                .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
+
+        start = System.currentTimeMillis();
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        manualRedactions.setImageRecategorizations(Set.of(ManualImageRecategorization.builder()
+                .id("37eee3e9d589a5cc529bfec38c3ba479")
+                .status(Status.APPROVED)
+                .type("signature")
+                .build()));
+
+        request.setManualRedactions(manualRedactions);
+
+        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+
+        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        end = System.currentTimeMillis();
+        System.out.println("reanalysis analysis duration: " + (end - start));
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+
+        deleted.remove("mouse");
+        reanlysisVersions.put("mouse", 4L);
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER)).thenReturn(4L);
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_DOSSIER_TEMPLATE_ID, DictionaryResource.GLOBAL_DOSSIER))
+                .thenReturn(getDictionaryResponse(VERTEBRATE, false));
+
+        analyzeService.reanalyze(request);
+
+        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        System.out.println("hi");
+    }
+
+    @Test
+    public void redactionTest2() throws IOException {
+
+        long start = System.currentTimeMillis();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/S16.pdf");
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
         request.setExcludedPages(Set.of(1));