Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-9116' into 'master'

Browse Source 
RED-9116 - Get the latest EntityLog entries for component rules execution

Closes RED-9116

See merge request redactmanager/redaction-service!392
This commit is contained in:
Andrei Isvoran 2024-05-07 15:16:23 +02:00
commit 730f6c601c
2 changed files with 83 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -105,13 +105,9 @@ public class AnalyzeService {
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
Set<Integer> sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
relevantManuallyModifiedAnnotationIds);
sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest,
document,
dictionaryIncrement,
importedRedactions,
relevantManuallyModifiedAnnotationIds));
analyzeRequest.getFileId(),
relevantManuallyModifiedAnnotationIds);
sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest, document, dictionaryIncrement, importedRedactions, relevantManuallyModifiedAnnotationIds));
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -130,7 +126,6 @@ public class AnalyzeService {
entityLogChanges,
document,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
true,
Collections.emptySet());
}
@ -180,7 +175,6 @@ public class AnalyzeService {
entityLogChanges,
document,
document.getNumberOfPages(),
dictionaryIncrement.getDictionaryVersion(),
true,
new HashSet<>(allFileAttributes));
}
@ -245,7 +239,6 @@ public class AnalyzeService {
entityLogChanges,
document,
document.getNumberOfPages(),
dictionary.getVersion(),
false,
new HashSet<>(allFileAttributes));
}
@ -257,7 +250,6 @@ public class AnalyzeService {
EntityLogChanges entityLogChanges,
Document document,
int numberOfPages,
DictionaryVersion dictionaryVersion,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes) {
@ -281,7 +273,7 @@ public class AnalyzeService {
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (entityLogChanges.hasChanges() || !isReanalysis) {
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog);
}
log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -314,15 +306,17 @@ public class AnalyzeService {
KieWrapper kieWrapperComponentRules,
Document document,
Set<FileAttribute> addedFileAttributes,
EntityLogChanges entityLogChanges,
DictionaryVersion dictionaryVersion) {
EntityLog entityLog) {
if (!kieWrapperComponentRules.isPresent()) {
return;
}
// We need the latest EntityLog entries for components rules execution
entityLog.getEntityLogEntry().addAll(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLogChanges.getEntityLog(),
entityLog,
document,
addedFileAttributes.stream()
.toList());
@ -350,11 +344,7 @@ public class AnalyzeService {
ImportedRedactions importedRedactions,
Set<String> relevantManuallyModifiedAnnotationIds) {
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
document,
analyzeRequest,
importedRedactions,
relevantManuallyModifiedAnnotationIds);
return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds);
}

75
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -1,12 +1,16 @@
package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
@ -28,6 +32,8 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
@ -150,10 +156,15 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertNotNull(componentLog);
var experimentalDates = componentLog.getComponentLogEntries().stream().filter(c -> c.getName().equals("Experimental_Starting_Date")).findFirst().get();
var experimentalDates = componentLog.getComponentLogEntries()
.stream()
.filter(c -> c.getName().equals("Experimental_Starting_Date"))
.findFirst()
.get();
assertNotNull(experimentalDates);
String dates = experimentalDates.getComponentValues().get(0).getValue();
String dates = experimentalDates.getComponentValues()
.get(0).getValue();
String[] dateArray = dates.split(", ");
boolean allEqual = true;
for (String date : dateArray) {
@ -215,4 +226,64 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@Test
public void testComponentLogWithManualRemoval() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
"files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactionClasspathResource.getInputStream());
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
System.out.println("Finished structure analysis");
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("Finished analysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var glpStudy = componentLog.getComponentLogEntries()
.stream()
.filter(componentLogEntry -> componentLogEntry.getName().equals("GLP_Study"))
.findFirst()
.get();
assertEquals("Yes",
glpStudy.getComponentValues()
.get(0).getValue());
var glpStudyAnnotations = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().contains("DOC.8.0"))
.toList();
Set<IdRemoval> idRemovalSet = new HashSet<>();
glpStudyAnnotations.forEach(glpStudyAnnotation -> {
idRemovalSet.add(IdRemoval.builder()
.annotationId(glpStudyAnnotation.getId())
.removeFromAllDossiers(false)
.removeFromDictionary(false)
.requestDate(OffsetDateTime.now())
.user("user")
.fileId(TEST_FILE_ID)
.build());
});
request.setManualRedactions(ManualRedactions.builder()
.idsToRemove(idRemovalSet)
.build());
analyzeService.reanalyze(request);
componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
glpStudy = componentLog.getComponentLogEntries()
.stream()
.filter(componentLogEntry -> componentLogEntry.getName().equals("GLP_Study"))
.findFirst()
.get();
assertEquals("No",
glpStudy.getComponentValues()
.get(0).getValue());
}
}