Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

reworked section stuff

Browse Source
This commit is contained in:
Timo Bejan 2021-11-03 16:18:38 +02:00
parent 806eeac19e
commit 6e4c2f244d
10 changed files with 261 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java
View File

@ -27,7 +27,11 @@ public class SectionArea {
private String header;
public boolean contains(Rectangle other) {
return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight();
return page == other.getPage()
&& topLeft.getX() <= other.getTopLeft().getX()
&& topLeft.getX() + getWidth() >= other.getTopLeft().getX() + other.getWidth()
&& getTopLeft().getY() <= other.getTopLeft().getY()
&& getTopLeft().getY() + getHeight() >= other.getTopLeft().getY() + other.getHeight();
}
}

17
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java
View File

@ -3,10 +3,9 @@ package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
@Data
@AllArgsConstructor
@ -15,6 +14,16 @@ public class SectionGrid {
private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
private Map<Integer, String> sectionHeadlines = new HashMap<>();
private List<SectionGridSection> sections = new ArrayList<>();
@Data
@RequiredArgsConstructor
public static class SectionGridSection {
private final int sectionNumber;
private final String headline;
private final Set<Integer> pages;
private final List<SectionArea> sectionAreas;
}
}

13
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java
View File

@ -1,10 +1,6 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.*;
import java.util.List;
@ -31,8 +27,11 @@ public class SectionRectangle {
private List<CellRectangle> tableCells;
public boolean contains(Rectangle other) {
return this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight();
return topLeft.getX() <= other.getTopLeft().getX()
&& topLeft.getX() + getWidth() >= other.getTopLeft().getX() + other.getWidth()
&& getTopLeft().getY() <= other.getTopLeft().getY()
&& getTopLeft().getY() + getHeight() >= other.getTopLeft().getY() + other.getHeight();
}
}

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -24,6 +24,7 @@ import org.springframework.web.bind.annotation.RestController;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.stream.Collectors;
@Slf4j
@RestController
@ -145,7 +146,7 @@ public class RedactionController implements RedactionResource {
}
@Override
public RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest) {
public RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest) {
log.info("Requested preview for: {}", redactionRequest);
dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());
@ -155,16 +156,19 @@ public class RedactionController implements RedactionResource {
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
if(redactionRequest.isWithSectionDataForManualRedactions()) {
if (redactionRequest.isWithSectionDataForManualRedactions()) {
sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
if (sectionGrid.getSectionHeadlines().isEmpty()) {
if (sectionGrid.getSections().isEmpty()) {
log.info("SectionGrid does not have headlines set. Computing headlines now!");
var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
// enhance section grid with headline data
for(var sectionText: text.getSectionTexts()){
sectionGrid.getSectionHeadlines().put(sectionText.getSectionNumber(), sectionText.getHeadline());
for (var sectionText : text.getSectionTexts()) {
sectionGrid.getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
sectionText.getHeadline(),
sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
sectionText.getSectionAreas()));
}
redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java
View File

@ -84,7 +84,13 @@ public class ReanalyzeService {
.getSectionText()));
// enhance section grid with headline data
classifiedDoc.getSectionText().forEach(st -> classifiedDoc.getSectionGrid().getSectionHeadlines().put(st.getSectionNumber(), st.getHeadline()));
classifiedDoc.getSectionText().forEach(sectionText -> {
classifiedDoc
.getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
sectionText.getHeadline(),
sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
sectionText.getSectionAreas()));
});
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
.getSectionGrid());

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -17,7 +17,7 @@ import java.util.stream.Collectors;
public class RedactionLogMergeService {
private final DictionaryService dictionaryService;
private final SectionTextService sectionTextService;
public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId,
ManualRedactions manualRedactions, Set<Integer> excludedPages) {
@ -217,7 +217,7 @@ public class RedactionLogMergeService {
}
public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
private List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments, String dossierTemplateId) {
@ -232,7 +232,7 @@ public class RedactionLogMergeService {
redactionLogEntry.setPositions(manualRedactionEntry.getPositions());
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId()));
handleSectionText(sectionGrid, redactionLogEntry);
sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
redactionLogEntries.add(redactionLogEntry);
@ -242,29 +242,6 @@ public class RedactionLogMergeService {
return redactionLogEntries;
}
private void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.getSection() != null) {
// set by UI
return;
}
if (sectionGrid != null) {
var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
if (firstPosition != null) {
List<SectionRectangle> pageRectangles = sectionGrid.getRectanglesPerPage().get(firstPosition.getPage());
for (var rectangle : pageRectangles) {
if (rectangle.contains(firstPosition)) {
redactionLogEntry.setSection(sectionGrid.getSectionHeadlines().get(rectangle.getPart()));
}
}
}
}
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {

37
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java Normal file
View File

@ -0,0 +1,37 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.SectionGrid;
import org.springframework.stereotype.Service;
@Service
public class SectionTextService {
public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.getSection() != null) {
// set by UI
return;
}
if (sectionGrid != null) {
var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
if (firstPosition != null) {
for (var section : sectionGrid.getSections()) {
if (section.getPages().contains(firstPosition.getPage())) {
for (var sectionArea : section.getSectionAreas()) {
if (sectionArea.contains(firstPosition)) {
redactionLogEntry.setSection(section.getHeadline());
redactionLogEntry.setSectionNumber(section.getSectionNumber());
return;
}
}
}
}
}
}
}
}

98
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SectionTextServiceTest.java Normal file
View File

@ -0,0 +1,98 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
import com.iqser.red.service.redaction.v1.server.redaction.service.SectionTextService;
import lombok.SneakyThrows;
import org.apache.commons.compress.utils.Lists;
import org.assertj.core.util.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.MOCK)
@Import(SectionTextServiceTest.TestConfiguration.class)
public class SectionTextServiceTest {
@Autowired
private ObjectMapper objectMapper;
@Autowired
private SectionTextService service;
@Autowired
private RedactionLogMergeService redactionLogMergeService;
@MockBean
private DictionaryService dictionaryService;
@MockBean
private AmazonS3 amazonS3;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
public static class TestConfiguration {
}
@Test
@SneakyThrows
public void testMerge(){
when(dictionaryService.getColor(Mockito.anyString(), Mockito.anyString())).thenReturn(new float[]{0,0,0});
RedactionLogEntry entry = objectMapper.readValue(new ClassPathResource("contains/entry.json").getInputStream(), RedactionLogEntry.class);
RedactionLog redactionLog = new RedactionLog(1, Lists.newArrayList(),Lists.newArrayList(),1,1,1,1);
SectionGrid sectionGrid = objectMapper.readValue(new ClassPathResource("contains/sections.json").getInputStream(), SectionGrid.class);
ManualRedactionEntry addRequest = new ManualRedactionEntry();
addRequest.setAddToDictionary(false);
addRequest.setAddToDossierDictionary(false);
addRequest.setStatus(Status.APPROVED);
addRequest.setReason("reason");
addRequest.setType("manual");
addRequest.setId("123");
addRequest.setUser("123");
addRequest.setPositions(entry.getPositions());
RedactionLog merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, "1", ManualRedactions.builder().entriesToAdd(Sets.newLinkedHashSet(addRequest)).build(), Sets.newHashSet());
assertThat(merged.getRedactionLogEntry().isEmpty()).isFalse();
assertThat(merged.getRedactionLogEntry().iterator().next().getSection()).isEqualTo("Headline 1");
assertThat(merged.getRedactionLogEntry().iterator().next().getSectionNumber()).isEqualTo(1);
}
@Test
@SneakyThrows
public void testSectionMatch() {
SectionGrid sectionGrid = objectMapper.readValue(new ClassPathResource("contains/sections.json").getInputStream(), SectionGrid.class);
RedactionLogEntry entry = objectMapper.readValue(new ClassPathResource("contains/entry.json").getInputStream(), RedactionLogEntry.class);
service.handleSectionText(sectionGrid, entry);
assertThat(entry.getSection()).isEqualTo("Headline 1");
assertThat(entry.getSectionNumber()).isEqualTo(1);
}
}

60
redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/entry.json Normal file
View File

@ -0,0 +1,60 @@
{
"id": "4092d4f880675587b33c8c1d4a7420fa",
"type": "manual",
"value": "NOTIFICATION OF AN ACTIVE\nSUBSTANCE UNDER COMMISSION\nREGULATION (EU) 844/2012",
"reason": "names and addresses of persons involved in testing on vertebrate animals",
"matchedRule": 0,
"legalBasis": "[Reg (EC) No 1107/2009 Art. 63 (2g)]",
"redacted": true,
"color": [
0.6745098,
0.9882353,
0
],
"positions": [
{
"topLeft": {
"x": 189.224,
"y": 478.632
},
"width": 231.03699,
"height": 22.912,
"page": 1
},
{
"topLeft": {
"x": 169.65797,
"y": 460.234
},
"width": 270.143,
"height": 22.912,
"page": 1
},
{
"topLeft": {
"x": 197.63498,
"y": 441.836
},
"width": 214.25,
"height": 22.912,
"page": 1
}
],
"manual": true,
"status": "APPROVED",
"manualRedactionType": "ADD",
"manualRedactionUserId": "7adf30b4-a0d8-4abb-adbb-760b1211098f",
"textBefore": null,
"textAfter": null,
"startOffset": 0,
"endOffset": 0,
"imageHasTransparency": false,
"excluded": false,
"recategorizationType": null,
"legalBasisChangeValue": null,
"hint": false,
"recommendation": false,
"image": false,
"dictionaryEntry": false,
"dossierDictionaryEntry": false
}

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/sections.json Normal file
View File

@ -0,0 +1,23 @@
{
"sections": [
{
"sectionNumber": 1,
"headline": "Headline 1",
"pages": [
1
],
"sectionAreas": [
{
"topLeft": {
"x": 169.65797,
"y": 282.43396
},
"width": 271.14307,
"height": 285.75897,
"page": 1,
"header": "Unknown"
}
]
}
]
}