Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6619 - fix integration-tests by adding versions and move the hasMinimumSize-logic into own boolea

Browse Source
This commit is contained in:
Thomas Beyer 2023-04-28 12:13:54 +02:00
parent fcc4085321
commit 4a76e89ab8
4 changed files with 23 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java
View File

@ -25,10 +25,11 @@ public class Cell extends Rectangle {
private boolean isHeaderCell;
private boolean minimumSize;
public Cell(Point2D topLeft, Point2D bottomRight) {
super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()));
this.minimumSize = (float) (bottomRight.getX() - topLeft.getX()) >= 1 && (float) (bottomRight.getY() - topLeft.getY()) >= 1;
}
@ -66,8 +67,4 @@ public class Cell extends Rectangle {
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
}
public boolean hasMinimumSize() {
return this.getHeight() >= 1 && this.getWidth() >= 1;
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java
View File

@ -261,7 +261,7 @@ public class Table extends AbstractTextContainer {
if (intersectionCell.isPresent()) {
cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
}
if (cell.hasMinimumSize()) {
if (cell.isMinimumSize()) {
row.add(cell);
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java
View File

@ -88,7 +88,7 @@ public class TableExtractionService {
for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
TextBlock textBlock = (TextBlock) abstractTextContainer;
for (Cell cell : cells) {
if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
if (cell.isMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
textBlock.getPdfMinY(),
textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@ -109,7 +109,7 @@ public class TableExtractionService {
List<Cell> overlappingCells = new ArrayList<>();
for (Cell c : cells) {
if (c.hasMinimumSize() && c.intersects(area)) {
if (c.isMinimumSize() && c.intersects(area)) {
overlappingCells.add(c);
}
}

51
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -3,20 +3,12 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.KieFileSystem;
import org.kie.api.builder.KieModule;
import org.kie.api.runtime.KieContainer;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
@ -43,31 +35,16 @@ import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
static class RedactionIntegrationTestConfiguration {
@Bean
public KieContainer kieContainer() {
KieServices kieServices = KieServices.Factory.get();
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
kieFileSystem.write("src/test/resources/drools/rules_v2", kieServices.getResources().newInputStreamResource(input));
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
kieBuilder.buildAll();
KieModule kieModule = kieBuilder.getKieModule();
return kieServices.newKieContainer(kieModule.getReleaseId());
}
@Bean
@Primary
public StorageService inmemoryStorage() {
@ -122,12 +99,17 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
AnalyzeRequest request = uploadFileToStorage("files/new/simplified2.pdf");
dictionary.clear();
dictionary.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller"));
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Dr. Alan Miller"));
String name = "Dr. Alan Miller";
String nameWithCompletionDate = "Dr. Alan Miller COMPLETION DATE:";
falsePositive.clear();
falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:"));
dictionary.get(DICTIONARY_AUTHOR).add(name);
dictionary.put(DICTIONARY_PII, Arrays.asList(name));
falsePositive.put(DICTIONARY_PII, Arrays.asList(nameWithCompletionDate));
reanlysisVersions.put(name, 1L);
reanlysisVersions.put(nameWithCompletionDate, 1L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
mockDictionaryCalls(0L);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
@ -139,7 +121,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo("Dr. Alan Miller");
assertThat(redactionLogEntry.getValue()).isEqualTo(name);
assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
@ -151,6 +133,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
}
/**
* The case in this test: The term 'Evans P.G.' is very close to a table-cell. It will get redacted nevertheless.
*/
@ -163,7 +146,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
dictionary.clear();
falsePositive.clear();
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList("Evans P.G."));
String name = "Evans P.G.";
dictionary.put(DICTIONARY_AUTHOR, Arrays.asList(name));
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
analyzeService.analyze(request);
@ -175,7 +159,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
assertThat(redactionLogEntry.getValue()).isEqualTo("Evans P.G.");
assertThat(redactionLogEntry.getValue()).isEqualTo(name);
assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
@ -186,4 +170,5 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
}
}