RED-6369: Rules Refactor
* fix pmd errors
This commit is contained in:
parent
3e5c7dafc0
commit
d978b2fb4a
@ -55,7 +55,7 @@ public class ConcatenatedTextBlock implements TextBlock {
|
|||||||
|
|
||||||
private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
|
private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
|
||||||
|
|
||||||
return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getBoundary().contains(stringIdx))).findAny().orElseThrow(IndexOutOfBoundsException::new);
|
return atomicTextBlocks.stream().filter(textBlock -> textBlock.getBoundary().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -192,12 +192,10 @@ public class EntityCreationService {
|
|||||||
boolean typeMatches = entityNode.getType().equals(previousEntity.getType());
|
boolean typeMatches = entityNode.getType().equals(previousEntity.getType());
|
||||||
boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType());
|
boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType());
|
||||||
boolean intersects = entityNode.intersects(previousEntity);
|
boolean intersects = entityNode.intersects(previousEntity);
|
||||||
if (typeMatches && entityTypeMatches && intersects) {
|
if (!typeMatches || !entityTypeMatches || !intersects) {
|
||||||
previousEntity = entityNode;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,135 +0,0 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
|
||||||
|
|
||||||
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.bBoxUnionAbstractTextContainer;
|
|
||||||
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle2D;
|
|
||||||
import static java.util.stream.Collectors.groupingBy;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.Section;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class ImageSortService {
|
|
||||||
|
|
||||||
public SortedImages sortImagesIntoStructure(Document document) {
|
|
||||||
|
|
||||||
SortedImages sortedImages = new SortedImages(new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>());
|
|
||||||
|
|
||||||
Map<Integer, List<PdfImage>> imagesByPage = document.getSections()
|
|
||||||
.stream()
|
|
||||||
.flatMap(section -> section.getImages().stream())
|
|
||||||
.distinct()
|
|
||||||
.collect(groupingBy(PdfImage::getPage));
|
|
||||||
|
|
||||||
for (int pageNumber : imagesByPage.keySet()) {
|
|
||||||
List<AbstractTextContainer> textContainersOnPage = document.getSections()
|
|
||||||
.stream()
|
|
||||||
.flatMap(section -> section.getPageBlocks().stream())
|
|
||||||
.filter(abstractTextContainer -> abstractTextContainer.getPage() == pageNumber)
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
List<Section> sectionsOnPage = document.getSections()
|
|
||||||
.stream()
|
|
||||||
.filter(section -> section.getPageBlocks().stream().anyMatch(block -> block.getPage() == pageNumber))
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
for (PdfImage image : imagesByPage.get(pageNumber)) {
|
|
||||||
sortImage(textContainersOnPage, sectionsOnPage, image, sortedImages);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sortedImages;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void sortImage(List<AbstractTextContainer> textContainersOnPage, List<Section> sectionsOnPage, PdfImage image, SortedImages sortedImages) {
|
|
||||||
|
|
||||||
Optional<AbstractTextContainer> containingTextContainer = getContainingTextContainer(image, textContainersOnPage);
|
|
||||||
Optional<Section> sectionContainingTextContainer = getContainingSection(image, sectionsOnPage);
|
|
||||||
List<AbstractTextContainer> containedTextContainers = getContainedTextContainers(image, textContainersOnPage);
|
|
||||||
List<Section> containedSections = getContainedSections(image, sectionsOnPage);
|
|
||||||
if (containingTextContainer.isPresent()) {
|
|
||||||
if (sortImageIntoTextContainerOrCell(image, sortedImages, containingTextContainer.get())) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static boolean sortImageIntoTextContainerOrCell(PdfImage image, SortedImages sortedImages, AbstractTextContainer containingTextContainer) {
|
|
||||||
|
|
||||||
if (containingTextContainer instanceof TextBlock) {
|
|
||||||
sortedImages.containedInTextContainer().computeIfAbsent(containingTextContainer, sortedImage -> new ArrayList<>()).add(image);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (containingTextContainer instanceof Table) {
|
|
||||||
Optional<Cell> containingCell = getContainingCell((Table) containingTextContainer, image);
|
|
||||||
if (containingCell.isPresent()) {
|
|
||||||
sortedImages.containedInCell().computeIfAbsent(containingCell.get(), sortedImage -> new ArrayList<>()).add(image);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static Optional<Cell> getContainingCell(Table table, PdfImage image) {
|
|
||||||
|
|
||||||
return table.getRows().stream().flatMap(List::stream).filter(cell -> cell.contains(toRectangle2D(image.getPosition()))).findFirst();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private List<Section> getContainedSections(PdfImage image, List<Section> sectionsOnPage) {
|
|
||||||
|
|
||||||
return sectionsOnPage.stream()
|
|
||||||
.filter(section -> toRectangle2D(image.getPosition()).contains(bBoxUnionAbstractTextContainer(section.getPageBlocks()
|
|
||||||
.stream()
|
|
||||||
.filter(block -> block.getPage() == image.getPage())
|
|
||||||
.toList())))
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private List<AbstractTextContainer> getContainedTextContainers(PdfImage image, List<AbstractTextContainer> textContainersOnPage) {
|
|
||||||
|
|
||||||
return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(image.getPosition()).contains(toRectangle2D(textContainer))).toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private Optional<Section> getContainingSection(PdfImage image, List<Section> sectionsOnPage) {
|
|
||||||
|
|
||||||
return sectionsOnPage.stream()//
|
|
||||||
.filter(section -> //
|
|
||||||
bBoxUnionAbstractTextContainer(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList())//
|
|
||||||
.contains(toRectangle2D(image.getPosition()))).findFirst();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private Optional<AbstractTextContainer> getContainingTextContainer(PdfImage image, List<AbstractTextContainer> textContainersOnPage) {
|
|
||||||
|
|
||||||
return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(textContainer).contains(toRectangle2D(image.getPosition()))).findFirst();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public record SortedImages(
|
|
||||||
Map<Cell, List<PdfImage>> containedInCell,
|
|
||||||
Map<AbstractTextContainer, List<PdfImage>> containedInTextContainer,
|
|
||||||
Map<Section, List<PdfImage>> containedInSection,
|
|
||||||
Map<PdfImage, List<AbstractTextContainer>> containedByImage,
|
|
||||||
Map<PdfImage, List<Section>> sectionContainedByImage) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -40,7 +40,7 @@ public class RedactionLogCreatorService {
|
|||||||
|
|
||||||
List<RedactionLogEntry> entries = new ArrayList<>();
|
List<RedactionLogEntry> entries = new ArrayList<>();
|
||||||
Set<String> processedIds = new HashSet<>();
|
Set<String> processedIds = new HashSet<>();
|
||||||
entityNodes.forEach((entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId))));
|
entityNodes.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)));
|
||||||
|
|
||||||
return entries;
|
return entries;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import java.io.File;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||||
|
import org.junit.jupiter.api.Disabled;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.springframework.core.io.ClassPathResource;
|
import org.springframework.core.io.ClassPathResource;
|
||||||
|
|
||||||
@ -17,6 +18,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
|
@Disabled
|
||||||
public void visualizeMetolachlor() {
|
public void visualizeMetolachlor() {
|
||||||
|
|
||||||
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
||||||
@ -30,6 +32,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
|
@Disabled
|
||||||
public void visualizeRotatedTestDocument() {
|
public void visualizeRotatedTestDocument() {
|
||||||
|
|
||||||
String filename = "files/new/RotateTestFileWithImages";
|
String filename = "files/new/RotateTestFileWithImages";
|
||||||
|
|||||||
@ -1,58 +0,0 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
|
||||||
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.core.io.ClassPathResource;
|
|
||||||
|
|
||||||
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.document.services.ImageSortService;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
|
||||||
|
|
||||||
import lombok.SneakyThrows;
|
|
||||||
|
|
||||||
public class ImageSortServiceTest extends AbstractTestWithDictionaries {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ImageSortService imageSortService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private PdfSegmentationService segmentationService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ImageService imageService;
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void assertImagesContainedInCells() {
|
|
||||||
|
|
||||||
ImageSortService.SortedImages sortedImages = getSortedImages("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
|
|
||||||
//assertEquals(9, sortedImages.containedInCell().size());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@SneakyThrows
|
|
||||||
private ImageSortService.SortedImages getSortedImages(String filename) {
|
|
||||||
|
|
||||||
if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
|
|
||||||
prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
|
|
||||||
} else {
|
|
||||||
prepareStorage(filename + ".pdf");
|
|
||||||
}
|
|
||||||
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
|
|
||||||
|
|
||||||
try (InputStream inputStream = fileResource.getInputStream()) {
|
|
||||||
Map<Integer, List<PdfImage>> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
|
|
||||||
Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
|
|
||||||
return imageSortService.sortImagesIntoStructure(classifiedDoc);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Loading…
x
Reference in New Issue
Block a user