code format, dependecy and test update, logging for reanalysis
This commit is contained in:
parent
1d4708ad13
commit
ba28a3e0d3
@ -32,7 +32,7 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.iqser.red</groupId>
|
<groupId>com.iqser.red</groupId>
|
||||||
<artifactId>platform-commons-dependency</artifactId>
|
<artifactId>platform-commons-dependency</artifactId>
|
||||||
<version>1.2.9</version>
|
<version>1.3.0</version>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|||||||
@ -2,11 +2,13 @@ package com.iqser.red.service.redaction.v1.server.parsing.model;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import org.apache.pdfbox.text.TextPosition;
|
import org.apache.pdfbox.text.TextPosition;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
public class RedTextPosition {
|
public class RedTextPosition {
|
||||||
|
|
||||||
private String textMatrix;
|
private String textMatrix;
|
||||||
|
|||||||
@ -3,19 +3,23 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
|
|||||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||||
import lombok.Value;
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@Value
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
public class CellValue {
|
public class CellValue {
|
||||||
|
|
||||||
private List<TextBlock> textBlocks;
|
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||||
|
|
||||||
private int rowSpanStart;
|
private int rowSpanStart;
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
|
|
||||||
|
|||||||
@ -128,6 +128,7 @@ public class ReanalyzeService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
|
||||||
|
|
||||||
if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
|
if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
|
||||||
redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
|
redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion());
|
||||||
|
|||||||
@ -12,11 +12,11 @@ import java.io.FileOutputStream;
|
|||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public class FilySystemBackedStorageService extends StorageService {
|
public class FileSystemBackedStorageService extends StorageService {
|
||||||
|
|
||||||
private Map<String, File> dataMap = new HashMap<>();
|
private final Map<String, File> dataMap = new HashMap<>();
|
||||||
|
|
||||||
public FilySystemBackedStorageService() {
|
public FileSystemBackedStorageService() {
|
||||||
super(null, null);
|
super(null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -44,7 +44,7 @@ public class FilySystemBackedStorageService extends StorageService {
|
|||||||
|
|
||||||
public void clearStorage() {
|
public void clearStorage() {
|
||||||
this.dataMap.forEach((k, v) -> {
|
this.dataMap.forEach((k, v) -> {
|
||||||
// v.delete();
|
v.delete();
|
||||||
});
|
});
|
||||||
this.dataMap.clear();
|
this.dataMap.clear();
|
||||||
}
|
}
|
||||||
@ -134,7 +134,7 @@ public class RedactionIntegrationTest {
|
|||||||
@Bean
|
@Bean
|
||||||
@Primary
|
@Primary
|
||||||
public StorageService inmemoryStorage() {
|
public StorageService inmemoryStorage() {
|
||||||
return new FilySystemBackedStorageService();
|
return new FileSystemBackedStorageService();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -142,8 +142,8 @@ public class RedactionIntegrationTest {
|
|||||||
|
|
||||||
@After
|
@After
|
||||||
public void cleanupStorage() {
|
public void cleanupStorage() {
|
||||||
if (this.storageService instanceof FilySystemBackedStorageService) {
|
if (this.storageService instanceof FileSystemBackedStorageService) {
|
||||||
((FilySystemBackedStorageService) this.storageService).clearStorage();
|
((FileSystemBackedStorageService) this.storageService).clearStorage();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -458,16 +458,6 @@ public class RedactionIntegrationTest {
|
|||||||
assertThat(result).isNotNull();
|
assertThat(result).isNotNull();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testXXX() {
|
|
||||||
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_08_Volume_3CA_B-6_2018-09-06.pdf");
|
|
||||||
MemoryStats.printMemoryStats();
|
|
||||||
AnalyzeResult result = redactionController.analyze(request);
|
|
||||||
assertThat(result).isNotNull();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
|
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
|
||||||
|
|||||||
@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
|
|||||||
|
|
||||||
import com.amazonaws.services.s3.AmazonS3;
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
import com.iqser.red.service.configuration.v1.api.model.*;
|
import com.iqser.red.service.configuration.v1.api.model.*;
|
||||||
import com.iqser.red.service.redaction.v1.server.FilySystemBackedStorageService;
|
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
|
||||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||||
@ -97,7 +97,7 @@ public class EntityRedactionServiceTest {
|
|||||||
@Bean
|
@Bean
|
||||||
@Primary
|
@Primary
|
||||||
public StorageService inmemoryStorage() {
|
public StorageService inmemoryStorage() {
|
||||||
return new FilySystemBackedStorageService();
|
return new FileSystemBackedStorageService();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user