RED-6619 - fix integration-tests by adding versions and move the hasMinimumSize-logic into own boolea

This commit is contained in:
Thomas Beyer 2023-04-28 12:13:54 +02:00
parent fcc4085321
commit 4a76e89ab8
4 changed files with 23 additions and 41 deletions

View File

@ -25,10 +25,11 @@ public class Cell extends Rectangle {
private boolean isHeaderCell; private boolean isHeaderCell;
private boolean minimumSize;
public Cell(Point2D topLeft, Point2D bottomRight) { public Cell(Point2D topLeft, Point2D bottomRight) {
super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY())); super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()));
this.minimumSize = (float) (bottomRight.getX() - topLeft.getX()) >= 1 && (float) (bottomRight.getY() - topLeft.getY()) >= 1;
} }
@ -66,8 +67,4 @@ public class Cell extends Rectangle {
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " "); return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
} }
public boolean hasMinimumSize() {
return this.getHeight() >= 1 && this.getWidth() >= 1;
}
} }

View File

@ -261,7 +261,7 @@ public class Table extends AbstractTextContainer {
if (intersectionCell.isPresent()) { if (intersectionCell.isPresent()) {
cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks()); cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
} }
if (cell.hasMinimumSize()) { if (cell.isMinimumSize()) {
row.add(cell); row.add(cell);
} }
} }

View File

@ -88,7 +88,7 @@ public class TableExtractionService {
for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) { for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
TextBlock textBlock = (TextBlock) abstractTextContainer; TextBlock textBlock = (TextBlock) abstractTextContainer;
for (Cell cell : cells) { for (Cell cell : cells) {
if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(), if (cell.isMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
textBlock.getPdfMinY(), textBlock.getPdfMinY(),
textBlock.getPdfMaxX() - textBlock.getPdfMinX(), textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
textBlock.getPdfMaxY() - textBlock.getPdfMinY())) { textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@ -109,7 +109,7 @@ public class TableExtractionService {
List<Cell> overlappingCells = new ArrayList<>(); List<Cell> overlappingCells = new ArrayList<>();
for (Cell c : cells) { for (Cell c : cells) {
if (c.hasMinimumSize() && c.intersects(area)) { if (c.isMinimumSize() && c.intersects(area)) {
overlappingCells.add(c); overlappingCells.add(c);
} }
} }

View File

@ -3,20 +3,12 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
@ -43,31 +35,16 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class) @Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest { public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules_v2.drl"); private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
@Configuration @Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)}) @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
static class RedactionIntegrationTestConfiguration { static class RedactionIntegrationTestConfiguration {
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/test/resources/drools/rules_v2", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
}
@Bean @Bean
@Primary @Primary
public StorageService inmemoryStorage() { public StorageService inmemoryStorage() {
@ -122,12 +99,17 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
AnalyzeRequest request = uploadFileToStorage("files/new/simplified2.pdf"); AnalyzeRequest request = uploadFileToStorage("files/new/simplified2.pdf");
dictionary.clear(); String name = "Dr. Alan Miller";
dictionary.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller")); String nameWithCompletionDate = "Dr. Alan Miller COMPLETION DATE:";
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Dr. Alan Miller"));
falsePositive.clear(); dictionary.get(DICTIONARY_AUTHOR).add(name);
falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:")); dictionary.put(DICTIONARY_PII, Arrays.asList(name));
falsePositive.put(DICTIONARY_PII, Arrays.asList(nameWithCompletionDate));
reanlysisVersions.put(name, 1L);
reanlysisVersions.put(nameWithCompletionDate, 1L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
mockDictionaryCalls(0L);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request); analyzeService.analyze(request);
@ -139,7 +121,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0); RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo("Dr. Alan Miller"); assertThat(redactionLogEntry.getValue()).isEqualTo(name);
assertThat(redactionLogEntry.isRedacted()).isEqualTo(true); assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false); assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false); assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
@ -151,6 +133,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
} }
/** /**
* The case in this test: The term 'Evans P.G.' is very close to a table-cell. It will get redacted nevertheless. * The case in this test: The term 'Evans P.G.' is very close to a table-cell. It will get redacted nevertheless.
*/ */
@ -163,7 +146,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
dictionary.clear(); dictionary.clear();
falsePositive.clear(); falsePositive.clear();
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Evans P.G.")); String name = "Evans P.G.";
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList(name));
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request); analyzeService.analyze(request);
@ -175,7 +159,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0); RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR); assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo("Evans P.G."); assertThat(redactionLogEntry.getValue()).isEqualTo(name);
assertThat(redactionLogEntry.isRedacted()).isEqualTo(true); assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false); assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false); assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
@ -186,4 +170,5 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
} }
} }