Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #145: removed long test, fixed NPE

Browse Source 
Merge in RED/redaction-service from redaction-system-queue-prep to master

* commit '3af121193214d250fe9dc58056f13a315d724184':
  fixed reanalyse condition
  fixed reanalyse condition
  removed long test, fixed NPE
This commit is contained in:
Timo Bejan 2021-04-21 17:29:41 +02:00
commit 08f3ee8544
5 changed files with 21 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java
View File

@ -13,11 +13,6 @@ public interface RedactionResource {
String RULE_SET_PARAMETER_NAME = "ruleSetId";
String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";
@PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest);
@PostMapping(value = "/reanalyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
AnalyzeResult reanalyze(@RequestBody AnalyzeRequest renalyzeRequest);
@PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -1,7 +1,10 @@
package com.iqser.red.service.redaction.v1.server.controller;
import com.iqser.red.service.file.management.v1.api.model.FileType;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
@ -9,7 +12,6 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@ -35,21 +37,9 @@ public class RedactionController implements RedactionResource {
private final DroolsExecutionService droolsExecutionService;
private final DictionaryService dictionaryService;
private final AnnotationService annotationService;
private final ReanalyzeService reanalyzeService;
private final PdfSegmentationService pdfSegmentationService;
private final RedactionStorageService redactionStorageService;
@Override
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
return reanalyzeService.analyze(analyzeRequest);
}
@Override
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
return reanalyzeService.reanalyze(analyzeRequest);
}
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java
View File

@ -84,7 +84,7 @@ public class ReanalyzeService {
var text = redactionStorageService.getText(analyzeRequest.getProjectId(), analyzeRequest.getFileId());
// not yet ready for reanalysis
if (text.getNumberOfPages() == 0) {
if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -99,11 +99,5 @@ public class RedactionStorageService {
return projectId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
}
public static String getStorageId(String userId, String projectId, String filename) {
return userId + "/" + projectId + "/" + filename;
}
}
}

27
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClien
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@ -83,6 +84,9 @@ public class RedactionIntegrationTest {
@Autowired
private RedactionController redactionController;
@Autowired
private ReanalyzeService reanalyzeService;
@Autowired
private ObjectMapper objectMapper;
@ -464,7 +468,7 @@ public class RedactionIntegrationTest {
public void test270Rotated() {
AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf");
MemoryStats.printMemoryStats();
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
assertThat(result).isNotNull();
}
@ -474,12 +478,13 @@ public class RedactionIntegrationTest {
public void testLargeScannedFileOOM() {
AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
MemoryStats.printMemoryStats();
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
assertThat(result).isNotNull();
}
@Test
@Ignore
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
long start = System.currentTimeMillis();
@ -495,7 +500,7 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(new FileInputStream((path)));
System.out.println("Redacting file : " + path.getName());
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
@ -513,7 +518,7 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(1L);
long rstart = System.currentTimeMillis();
redactionController.reanalyze(request);
reanalyzeService.reanalyze(request);
long rend = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (rend - rstart));
@ -552,7 +557,7 @@ public class RedactionIntegrationTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID);
var text = redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID);
@ -606,7 +611,7 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
start = System.currentTimeMillis();
AnalyzeResult reanalyzeResult = redactionController.reanalyze(request);
AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request);
end = System.currentTimeMillis();
System.out.println("reanalysis analysis duration: " + (end - start));
@ -630,7 +635,7 @@ public class RedactionIntegrationTest {
long start = System.currentTimeMillis();
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID)
@ -692,7 +697,7 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
request.setManualRedactions(manualRedactions);
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
@ -700,7 +705,7 @@ public class RedactionIntegrationTest {
.status(Status.APPROVED)
.build()));
redactionController.reanalyze(request);
reanalyzeService.reanalyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
@ -815,7 +820,7 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID);
@ -862,7 +867,7 @@ public class RedactionIntegrationTest {
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
AnalyzeResult result = redactionController.analyze(request);
AnalyzeResult result = reanalyzeService.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.projectId(TEST_PROJECT_ID)