From 567bb00377747fb78f79da18373e812852b9a23c Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Apr 2021 18:11:01 +0300 Subject: [PATCH 1/3] removed long test, fixed NPE --- .../v1/resources/RedactionResource.java | 5 ----- .../server/controller/RedactionController.java | 18 ++++-------------- .../redaction/service/ReanalyzeService.java | 2 +- .../storage/RedactionStorageService.java | 6 ------ .../v1/server/RedactionIntegrationTest.java | 1 + 5 files changed, 6 insertions(+), 26 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 7b6c1952..de766ba5 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -13,11 +13,6 @@ public interface RedactionResource { String RULE_SET_PARAMETER_NAME = "ruleSetId"; String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}"; - @PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest); - - @PostMapping(value = "/reanalyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - AnalyzeResult reanalyze(@RequestBody AnalyzeRequest renalyzeRequest); @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 63a212b8..71102747 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -1,7 +1,10 @@ package com.iqser.red.service.redaction.v1.server.controller; import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.AnnotateRequest; +import com.iqser.red.service.redaction.v1.model.AnnotateResponse; +import com.iqser.red.service.redaction.v1.model.RedactionRequest; +import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; @@ -9,7 +12,6 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; @@ -35,21 +37,9 @@ public class RedactionController implements RedactionResource { private final DroolsExecutionService droolsExecutionService; private final DictionaryService dictionaryService; private final AnnotationService annotationService; - private final ReanalyzeService reanalyzeService; private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; - @Override - public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) { - return reanalyzeService.analyze(analyzeRequest); - - } - - @Override - public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { - return reanalyzeService.reanalyze(analyzeRequest); - } - public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index dfb14446..9fa3a8f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -84,7 +84,7 @@ public class ReanalyzeService { var text = redactionStorageService.getText(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); // not yet ready for reanalysis - if (text.getNumberOfPages() == 0) { + if (text == null || text.getNumberOfPages() == 0) { return analyze(analyzeRequest); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 2c96bd05..f350b7ac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -99,11 +99,5 @@ public class RedactionStorageService { return projectId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); } - - public static String getStorageId(String userId, String projectId, String filename) { - - return userId + "/" + projectId + "/" + filename; - } - } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 23a60d2f..c63151be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -480,6 +480,7 @@ public class RedactionIntegrationTest { @Test + @Ignore public void noExceptionShouldBeThrownForAnyFiles() throws IOException { long start = System.currentTimeMillis(); From c1d5be8f63b61c77183e0859c8dc5e1e9fce262c Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Apr 2021 18:11:58 +0300 Subject: [PATCH 2/3] fixed reanalyse condition --- .../redaction/v1/server/redaction/service/ReanalyzeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 9fa3a8f5..0a3e46f6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -84,7 +84,7 @@ public class ReanalyzeService { var text = redactionStorageService.getText(analyzeRequest.getProjectId(), analyzeRequest.getFileId()); // not yet ready for reanalysis - if (text == null || text.getNumberOfPages() == 0) { + if (redactionLog == null || text == null || text.getNumberOfPages() == 0) { return analyze(analyzeRequest); } From 3af121193214d250fe9dc58056f13a315d724184 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Apr 2021 18:16:39 +0300 Subject: [PATCH 3/3] fixed reanalyse condition --- .../v1/server/RedactionIntegrationTest.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index c63151be..821e20d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClien import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.memory.MemoryStats; +import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -83,6 +84,9 @@ public class RedactionIntegrationTest { @Autowired private RedactionController redactionController; + @Autowired + private ReanalyzeService reanalyzeService; + @Autowired private ObjectMapper objectMapper; @@ -464,7 +468,7 @@ public class RedactionIntegrationTest { public void test270Rotated() { AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf"); MemoryStats.printMemoryStats(); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); assertThat(result).isNotNull(); } @@ -474,7 +478,7 @@ public class RedactionIntegrationTest { public void testLargeScannedFileOOM() { AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); MemoryStats.printMemoryStats(); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); assertThat(result).isNotNull(); } @@ -496,7 +500,7 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(new FileInputStream((path))); System.out.println("Redacting file : " + path.getName()); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); Map> duplicates = new HashMap<>(); @@ -514,7 +518,7 @@ public class RedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(1L); long rstart = System.currentTimeMillis(); - redactionController.reanalyze(request); + reanalyzeService.reanalyze(request); long rend = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (rend - rstart)); @@ -553,7 +557,7 @@ public class RedactionIntegrationTest { ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf"); AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); var text = redactionStorageService.getText(TEST_PROJECT_ID, TEST_FILE_ID); @@ -607,7 +611,7 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE)); start = System.currentTimeMillis(); - AnalyzeResult reanalyzeResult = redactionController.reanalyze(request); + AnalyzeResult reanalyzeResult = reanalyzeService.reanalyze(request); end = System.currentTimeMillis(); System.out.println("reanalysis analysis duration: " + (end - start)); @@ -631,7 +635,7 @@ public class RedactionIntegrationTest { long start = System.currentTimeMillis(); AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() .projectId(TEST_PROJECT_ID) @@ -693,7 +697,7 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); request.setManualRedactions(manualRedactions); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() @@ -701,7 +705,7 @@ public class RedactionIntegrationTest { .status(Status.APPROVED) .build())); - redactionController.reanalyze(request); + reanalyzeService.reanalyze(request); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() @@ -816,7 +820,7 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); var redactionLog = redactionStorageService.getRedactionLog(TEST_PROJECT_ID, TEST_FILE_ID); @@ -863,7 +867,7 @@ public class RedactionIntegrationTest { AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); - AnalyzeResult result = redactionController.analyze(request); + AnalyzeResult result = reanalyzeService.analyze(request); AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() .projectId(TEST_PROJECT_ID)