diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java index 07e67c9f..6629de3e 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java @@ -27,7 +27,11 @@ public class SectionArea { private String header; public boolean contains(Rectangle other) { - return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight(); + return page == other.getPage() + && topLeft.getX() <= other.getTopLeft().getX() + && topLeft.getX() + getWidth() >= other.getTopLeft().getX() + other.getWidth() + && getTopLeft().getY() <= other.getTopLeft().getY() + && getTopLeft().getY() + getHeight() >= other.getTopLeft().getY() + other.getHeight(); } } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java index 3da17db0..d1f027a1 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java @@ -3,10 +3,9 @@ package com.iqser.red.service.redaction.v1.model; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Data @AllArgsConstructor @@ -15,6 +14,16 @@ public class SectionGrid { private Map> rectanglesPerPage = new HashMap<>(); - private Map sectionHeadlines = new HashMap<>(); + private List sections = new ArrayList<>(); + @Data + @RequiredArgsConstructor + public static class SectionGridSection { + + private final int sectionNumber; + private final String headline; + private final Set pages; + private final List sectionAreas; + + } } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java index 9ae38163..ccdebcb5 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java @@ -1,10 +1,6 @@ package com.iqser.red.service.redaction.v1.model; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; +import lombok.*; import java.util.List; @@ -31,8 +27,11 @@ public class SectionRectangle { private List tableCells; - public boolean contains(Rectangle other) { - return this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight(); + return topLeft.getX() <= other.getTopLeft().getX() + && topLeft.getX() + getWidth() >= other.getTopLeft().getX() + other.getWidth() + && getTopLeft().getY() <= other.getTopLeft().getY() + && getTopLeft().getY() + getHeight() >= other.getTopLeft().getY() + other.getHeight(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index e4592e29..3268dac0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -24,6 +24,7 @@ import org.springframework.web.bind.annotation.RestController; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.stream.Collectors; @Slf4j @RestController @@ -145,7 +146,7 @@ public class RedactionController implements RedactionResource { } @Override - public RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest) { + public RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest) { log.info("Requested preview for: {}", redactionRequest); dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); @@ -155,16 +156,19 @@ public class RedactionController implements RedactionResource { var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); - if(redactionRequest.isWithSectionDataForManualRedactions()) { + if (redactionRequest.isWithSectionDataForManualRedactions()) { sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId()); - if (sectionGrid.getSectionHeadlines().isEmpty()) { + if (sectionGrid.getSections().isEmpty()) { log.info("SectionGrid does not have headlines set. Computing headlines now!"); var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId()); // enhance section grid with headline data - for(var sectionText: text.getSectionTexts()){ - sectionGrid.getSectionHeadlines().put(sectionText.getSectionNumber(), sectionText.getHeadline()); + for (var sectionText : text.getSectionTexts()) { + sectionGrid.getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), + sectionText.getHeadline(), + sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), + sectionText.getSectionAreas())); } redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 3d017856..4a22087e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -84,7 +84,13 @@ public class ReanalyzeService { .getSectionText())); // enhance section grid with headline data - classifiedDoc.getSectionText().forEach(st -> classifiedDoc.getSectionGrid().getSectionHeadlines().put(st.getSectionNumber(), st.getHeadline())); + classifiedDoc.getSectionText().forEach(sectionText -> { + classifiedDoc + .getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), + sectionText.getHeadline(), + sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), + sectionText.getSectionAreas())); + }); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc .getSectionGrid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index da0837d6..423c0dfa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -17,7 +17,7 @@ import java.util.stream.Collectors; public class RedactionLogMergeService { private final DictionaryService dictionaryService; - + private final SectionTextService sectionTextService; public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId, ManualRedactions manualRedactions, Set excludedPages) { @@ -217,7 +217,7 @@ public class RedactionLogMergeService { } - public List addManualAddEntries(SectionGrid sectionGrid, + private List addManualAddEntries(SectionGrid sectionGrid, Set manualAdds, Map> comments, String dossierTemplateId) { @@ -232,7 +232,7 @@ public class RedactionLogMergeService { redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); - handleSectionText(sectionGrid, redactionLogEntry); + sectionTextService.handleSectionText(sectionGrid, redactionLogEntry); redactionLogEntries.add(redactionLogEntry); @@ -242,29 +242,6 @@ public class RedactionLogMergeService { return redactionLogEntries; } - private void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) { - - if (redactionLogEntry.getSection() != null) { - // set by UI - return; - } - - if (sectionGrid != null) { - var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null; - - if (firstPosition != null) { - List pageRectangles = sectionGrid.getRectanglesPerPage().get(firstPosition.getPage()); - - for (var rectangle : pageRectangles) { - if (rectangle.contains(firstPosition)) { - redactionLogEntry.setSection(sectionGrid.getSectionHeadlines().get(rectangle.getPart())); - } - } - - } - } - } - private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java new file mode 100644 index 00000000..82fa5428 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionGrid; +import org.springframework.stereotype.Service; + +@Service +public class SectionTextService { + + + public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) { + + if (redactionLogEntry.getSection() != null) { + // set by UI + return; + } + + if (sectionGrid != null) { + var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null; + + if (firstPosition != null) { + + for (var section : sectionGrid.getSections()) { + if (section.getPages().contains(firstPosition.getPage())) { + for (var sectionArea : section.getSectionAreas()) { + if (sectionArea.contains(firstPosition)) { + redactionLogEntry.setSection(section.getHeadline()); + redactionLogEntry.setSectionNumber(section.getSectionNumber()); + return; + } + } + } + } + } + } + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SectionTextServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SectionTextServiceTest.java new file mode 100644 index 00000000..7f253a8d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SectionTextServiceTest.java @@ -0,0 +1,98 @@ +package com.iqser.red.service.redaction.v1.server.redaction.utils; + +import com.amazonaws.services.s3.AmazonS3; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.server.Application; +import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService; +import com.iqser.red.service.redaction.v1.server.redaction.service.SectionTextService; +import lombok.SneakyThrows; +import org.apache.commons.compress.utils.Lists; +import org.assertj.core.util.Sets; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.ClassPathResource; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.MOCK) +@Import(SectionTextServiceTest.TestConfiguration.class) +public class SectionTextServiceTest { + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private SectionTextService service; + + @Autowired + private RedactionLogMergeService redactionLogMergeService; + + @MockBean + private DictionaryService dictionaryService; + + @MockBean + private AmazonS3 amazonS3; + + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + public static class TestConfiguration { + + } + + + @Test + @SneakyThrows + public void testMerge(){ + + when(dictionaryService.getColor(Mockito.anyString(), Mockito.anyString())).thenReturn(new float[]{0,0,0}); + + RedactionLogEntry entry = objectMapper.readValue(new ClassPathResource("contains/entry.json").getInputStream(), RedactionLogEntry.class); + + RedactionLog redactionLog = new RedactionLog(1, Lists.newArrayList(),Lists.newArrayList(),1,1,1,1); + SectionGrid sectionGrid = objectMapper.readValue(new ClassPathResource("contains/sections.json").getInputStream(), SectionGrid.class); + ManualRedactionEntry addRequest = new ManualRedactionEntry(); + + addRequest.setAddToDictionary(false); + addRequest.setAddToDossierDictionary(false); + addRequest.setStatus(Status.APPROVED); + addRequest.setReason("reason"); + addRequest.setType("manual"); + addRequest.setId("123"); + addRequest.setUser("123"); + addRequest.setPositions(entry.getPositions()); + RedactionLog merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, "1", ManualRedactions.builder().entriesToAdd(Sets.newLinkedHashSet(addRequest)).build(), Sets.newHashSet()); + + assertThat(merged.getRedactionLogEntry().isEmpty()).isFalse(); + + assertThat(merged.getRedactionLogEntry().iterator().next().getSection()).isEqualTo("Headline 1"); + assertThat(merged.getRedactionLogEntry().iterator().next().getSectionNumber()).isEqualTo(1); + } + + @Test + @SneakyThrows + public void testSectionMatch() { + + SectionGrid sectionGrid = objectMapper.readValue(new ClassPathResource("contains/sections.json").getInputStream(), SectionGrid.class); + + RedactionLogEntry entry = objectMapper.readValue(new ClassPathResource("contains/entry.json").getInputStream(), RedactionLogEntry.class); + + service.handleSectionText(sectionGrid, entry); + + assertThat(entry.getSection()).isEqualTo("Headline 1"); + assertThat(entry.getSectionNumber()).isEqualTo(1); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/entry.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/entry.json new file mode 100644 index 00000000..85ebb5d9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/entry.json @@ -0,0 +1,60 @@ +{ + "id": "4092d4f880675587b33c8c1d4a7420fa", + "type": "manual", + "value": "NOTIFICATION OF AN ACTIVE\nSUBSTANCE UNDER COMMISSION\nREGULATION (EU) 844/2012", + "reason": "names and addresses of persons involved in testing on vertebrate animals", + "matchedRule": 0, + "legalBasis": "[Reg (EC) No 1107/2009 Art. 63 (2g)]", + "redacted": true, + "color": [ + 0.6745098, + 0.9882353, + 0 + ], + "positions": [ + { + "topLeft": { + "x": 189.224, + "y": 478.632 + }, + "width": 231.03699, + "height": 22.912, + "page": 1 + }, + { + "topLeft": { + "x": 169.65797, + "y": 460.234 + }, + "width": 270.143, + "height": 22.912, + "page": 1 + }, + { + "topLeft": { + "x": 197.63498, + "y": 441.836 + }, + "width": 214.25, + "height": 22.912, + "page": 1 + } + ], + "manual": true, + "status": "APPROVED", + "manualRedactionType": "ADD", + "manualRedactionUserId": "7adf30b4-a0d8-4abb-adbb-760b1211098f", + "textBefore": null, + "textAfter": null, + "startOffset": 0, + "endOffset": 0, + "imageHasTransparency": false, + "excluded": false, + "recategorizationType": null, + "legalBasisChangeValue": null, + "hint": false, + "recommendation": false, + "image": false, + "dictionaryEntry": false, + "dossierDictionaryEntry": false +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/sections.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/sections.json new file mode 100644 index 00000000..994c46b3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/contains/sections.json @@ -0,0 +1,23 @@ +{ + "sections": [ + { + "sectionNumber": 1, + "headline": "Headline 1", + "pages": [ + 1 + ], + "sectionAreas": [ + { + "topLeft": { + "x": 169.65797, + "y": 282.43396 + }, + "width": 271.14307, + "height": 285.75897, + "page": 1, + "header": "Unknown" + } + ] + } + ] +}