Fix pmd

Merge in RED/redaction-service from RED-1039 to master

* commit '00b0cb160342f1857ac0e523f994918057d5fc6b':
  RED-1039: Fixed finding textpositions, RED-1042: Fixed get rectangles per line

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -10,6 +10,9 @@
     </parent>
 
     <artifactId>redaction-service-server-v1</artifactId>
+    <properties>
+        <slf4j.version>1.7.30</slf4j.version>
+    </properties>
 
     <dependencies>
         <dependency>
@@ -68,6 +71,11 @@
             <groupId>org.apache.pdfbox</groupId>
             <artifactId>pdfbox-tools</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>${slf4j.version}</version>
+        </dependency>
         <!-- spring -->
         <dependency>
             <groupId>org.springframework.cloud</groupId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -51,6 +51,9 @@ public class RedactionController implements RedactionResource {
     @Override
     public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
 
+        log.info("Starting redaction analysis...");
+        long startTime = System.currentTimeMillis();
+
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
             pdDocument.setAllSecurityToBeRemoved(true);
 
@@ -64,6 +67,8 @@ public class RedactionController implements RedactionResource {
 
             log.info("Redaction analysis successful...");
 
+            log.debug("Finishing redaction analysis took: {}.", System.currentTimeMillis() - startTime);
+
             return AnalyzeResult.builder()
                     .sectionGrid(classifiedDoc.getSectionGrid())
                     .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -32,6 +32,7 @@ public class SearchableText {
 
     }
 
+
     @SuppressWarnings("checkstyle:ModifiedControlVariable")
     public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
                                                      List<TextPositionSequence> sequencesSubList) {
@@ -66,9 +67,12 @@ public class SearchableText {
             for (int j = 0; j < searchSpace.get(i).length(); j++) {
 
                 if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
-                        .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
-                        .charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
-                    if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
+                        .charAt(searchSpace.get(i - 1)
+                                .length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && searchSpace.get(i)
+                        .charAt(j - 1, caseInsensitive) == ' ') {
+                    if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions()
+                            .isEmpty()) {
                         crossSequenceParts.add(partMatch);
                     }
                     continue;
@@ -80,8 +84,8 @@ public class SearchableText {
                     counter++;
                 }
 
-                if (searchSpace.get(i)
-                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
+                if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace
+                        .get(i)
                         .charAt(j, caseInsensitive) == '-') {
 
                     if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
@@ -100,14 +104,15 @@ public class SearchableText {
                     if (counter == searchString.length()) {
                         crossSequenceParts.add(partMatch);
 
-                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
-                                .length() - 1 && isSeparator(searchSpace.get(i)
+                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
+                                .length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
                                 .charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
                                 .length() - 1 && isSeparator(searchSpace.get(i + 1)
-                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
+                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i)
+                                .length() - 1 && searchSpace.get(i)
                                 .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
                                 .charAt(0, caseInsensitive) != ' ') {
-                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
+                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
                         }
 
                         counter = 0;
@@ -130,15 +135,21 @@ public class SearchableText {
         }
 
         return finalMatches;
-
     }
 
 
-    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
+    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts,
+                                                                     String searchString) {
+
+        List<EntityPositionSequence> result = new ArrayList<>();
+        String asString = buildString(crossSequenceParts);
+        if (!asString.equalsIgnoreCase(searchString)) {
+            return result;
+        }
 
         String plainId = IdBuilder.buildId(crossSequenceParts);
         String id = plainId;
-        List<EntityPositionSequence> result = new ArrayList<>();
+
         int currentPage = -1;
         int idDiffentPageSuffix = 1;
         EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
@@ -173,6 +184,12 @@ public class SearchableText {
     @Override
     public String toString() {
 
+        return buildString(sequences);
+    }
+
+
+    public String buildString(List<TextPositionSequence> sequences) {
+
         StringBuilder sb = new StringBuilder();
 
         TextPositionSequence previous = null;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java

@@ -30,7 +30,9 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 public class AnnotationService {
@@ -40,6 +42,9 @@ public class AnnotationService {
 
     public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
 
+        log.info("Annotating document...");
+        long startTime = System.currentTimeMillis();
+
         Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
 
         for (int page = 1; page <= document.getNumberOfPages(); page++) {
@@ -56,6 +61,7 @@ public class AnnotationService {
                 addAnnotations(logEntries, pdPage, page);
             }
         }
+        log.info("Finishing document annotation took: {}.", System.currentTimeMillis() - startTime);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -50,14 +50,23 @@ public class EntityRedactionService {
 
     public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
 
+        log.info("Processing document...");
+        long startTime = System.currentTimeMillis();
+
         dictionaryService.updateDictionary(ruleSetId);
         KieContainer container = droolsExecutionService.updateRules(ruleSetId);
         long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);
 
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
 
+        long current = System.currentTimeMillis() - startTime;
+        log.info("Adding image entries took: {}.", current);
+
         Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
 
+        current = System.currentTimeMillis() - startTime - current;
+        log.info("Finding entries took: {}.", current);
+
         if (dictionary.hasLocalEntries()) {
 
             Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
@@ -71,6 +80,9 @@ public class EntityRedactionService {
             Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
             EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
             EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
+
+            current = System.currentTimeMillis() - startTime - current;
+            log.info("Adding local dictionary annotations took: {}.", current);
         }
 
         for (Entity entity : documentEntities) {
@@ -89,10 +101,19 @@ public class EntityRedactionService {
             }
         }
 
+        current = System.currentTimeMillis() - startTime - current;
+        log.info("Adding entity position sequence took: {}.", current);
+
         dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
 
         classifiedDoc.setDictionaryVersion(dictionary.getVersion());
         classifiedDoc.setRulesVersion(rulesVersion);
+
+        current = System.currentTimeMillis() - startTime - current;
+        log.info("Adjusting rules set took: {}.", current);
+
+        log.info("Finishing document processing took: {}.", System.currentTimeMillis() - startTime);
+
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -34,7 +34,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 public class RedactionLogCreatorService {
@@ -47,6 +49,9 @@ public class RedactionLogCreatorService {
     public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
                                    String ruleSetId) {
 
+        log.info("Creating redaction log...");
+        long startTime = System.currentTimeMillis();
+
         Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
 
         for (int page = 1; page <= numberOfPages; page++) {
@@ -57,14 +62,25 @@ public class RedactionLogCreatorService {
                 addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
             }
 
+            long current = System.currentTimeMillis() - startTime;
+            log.info("Adding entries took: {}.", current);
+
             if (manualRedactionPages.contains(page)) {
                 addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
             }
 
+            current = System.currentTimeMillis() - startTime - current;
+            log.info("Adding manual entries took: {}.", current);
+
             if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
                 addImageEntries(classifiedDoc, page, ruleSetId);
             }
+
+            current = System.currentTimeMillis() - startTime - current;
+            log.info("Adding image entries took: {}.", current);
         }
+
+        log.info("Finishing redaction log creation took: {}.", System.currentTimeMillis() - startTime);
     }
 
 
@@ -194,7 +210,7 @@ public class RedactionLogCreatorService {
                     startIndex = i;
                 }
             }
-            if (startIndex != textPositions.size() - 1) {
+            if (startIndex != textPositions.size()) {
                 rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -39,6 +39,9 @@ public class PdfSegmentationService {
 
     public Document parseDocument(PDDocument pdDocument) throws IOException {
 
+        log.info("Parsing document...");
+        long startTime = System.currentTimeMillis();
+
         Document document = new Document();
 
         List<Page> pages = new ArrayList<>();
@@ -91,6 +94,8 @@ public class PdfSegmentationService {
 
         sectionsBuilderService.buildSections(document);
 
+        log.info("Finishing document parsing took: {}.", System.currentTimeMillis() - startTime);
+
         return document;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -19,11 +19,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
 @Service
 public class SectionsBuilderService {
 
     public void buildSections(Document document) {
 
+        log.debug("Building sections...");
+        long startTime = System.currentTimeMillis();
+
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
         List<Paragraph> chunkBlockList = new ArrayList<>();
         List<Header> headers = new ArrayList<>();
@@ -87,6 +93,9 @@ public class SectionsBuilderService {
         document.setParagraphs(chunkBlockList);
         document.setHeaders(headers);
         document.setFooters(footers);
+
+        log.debug("Finishing section building took: {}.", System.currentTimeMillis() - startTime);
+
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java

@@ -15,11 +15,16 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRuli
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
 @Service
 public class RulingCleaningService {
 
     public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
 
+        log.debug("Getting clean rulings...");
+
         if (!rulings.isEmpty()) {
             snapPoints(rulings, minCharWidth, maxCharHeight);
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java

@@ -23,11 +23,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
 @Service
 public class TableExtractionService {
 
     public void extractTables(CleanRulings cleanRulings, Page page) {
 
+        log.debug("Extracting tables...");
+        long startTime = System.currentTimeMillis();
+
         List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
         List<TextBlock> toBeRemoved = new ArrayList<>();
@@ -78,6 +84,9 @@ public class TableExtractionService {
         }
 
         page.getTextBlocks().removeAll(toBeRemoved);
+
+        log.debug("Finishing table extraction took: {}.", System.currentTimeMillis() - startTime);
+
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -418,6 +418,7 @@ public class RedactionIntegrationTest {
     }
 
 
+
     private List<File> getPathsRecursively(File path) {
 
         List<File> result = new ArrayList<>();
@@ -439,9 +440,16 @@ public class RedactionIntegrationTest {
     @Test
     public void redactionTest() throws IOException {
 
+//        49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
+//        182 Fludioxonil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
+//        38 A14325E - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf
+//        91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
+//        95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf
+
+
         System.out.println("redactionTest");
         long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
 
         AnalyzeRequest request = AnalyzeRequest.builder()
                 .ruleSetId(TEST_RULESET_ID)