Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #229: upgraded ps integration

Browse Source 
Merge in RED/redaction-service from persistence-service-integration to master

* commit '530747fa20bf5beb95a6b3d167cbfe09f0d145d5':
  upgraded ps integration
  upgraded ps integration
This commit is contained in:
Timo Bejan 2021-09-28 14:53:46 +02:00
commit 6d50fe1d06
16 changed files with 112 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/pom.xml
View File

@ -20,7 +20,7 @@
<dependency> <dependency>
<groupId>com.iqser.red.service</groupId> <groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-api-v1</artifactId> <artifactId>persistence-service-api-v1</artifactId>
<version>0.4.0</version> <version>0.15.0</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.model; package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@ -11,7 +12,6 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
@Data @Data
@Builder @Builder

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java
View File

@ -1,11 +1,11 @@
package com.iqser.red.service.redaction.v1.model; package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.LegalBasis;
@Data @Data
@AllArgsConstructor @AllArgsConstructor

5
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.model; package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@ -11,7 +13,6 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
@Data @Data
@Builder @Builder
@ -45,7 +46,7 @@ public class RedactionLogEntry {
private String textAfter; private String textAfter;
@Builder.Default @Builder.Default
private List<com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment> comments = new ArrayList<>(); private List<Comment> comments = new ArrayList<>();
private int startOffset; private int startOffset;
private int endOffset; private int endOffset;

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java
View File

@ -3,8 +3,8 @@ package com.iqser.red.service.redaction.v1.model;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.redaction.v1.server.controller; package com.iqser.red.service.redaction.v1.server.controller;
import com.iqser.red.service.persistence.service.v1.api.model.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.resources.RedactionResource;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toSet;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import lombok.Data; import lombok.Data;
import lombok.Getter; import lombok.Getter;
@ -10,7 +11,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
@Data @Data
public class Dictionary { public class Dictionary {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
@ -8,7 +9,6 @@ import java.io.Serializable;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
@Data @Data
@AllArgsConstructor @AllArgsConstructor

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -8,17 +8,12 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationId;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
@ -245,11 +240,11 @@ public class AnalyzeService {
return Stream.concat(manualRedactions.getLegalBasisChanges() return Stream.concat(manualRedactions.getLegalBasisChanges()
.stream() .stream()
.map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorization() .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
.stream() .stream()
.map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove() .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
.stream() .stream()
.map(IdRemoval::getId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getId)))).map(AnnotationId::getId) .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} }

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.model.*;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
@ -12,7 +13,6 @@ import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.awt.Color; import java.awt.Color;
@ -28,8 +28,6 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor @RequiredArgsConstructor
public class AnnotationService { public class AnnotationService {
private final DictionaryService dictionaryService;
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException { public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.redaction.service; package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.model.*;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -9,13 +9,13 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@ -89,7 +89,7 @@ public class EntityRedactionService {
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
.getImageRecategorization()) { .getImageRecategorization()) {
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getId().getId() if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
.equals(imageId)) { .equals(imageId)) {
image.setType(imageRecategorization.getType()); image.setType(imageRecategorization.getType());
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/NerAnalyserService.java
View File

@ -2,10 +2,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.FileType;
import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient; import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection;

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -8,16 +8,9 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
@ -79,25 +72,25 @@ public class RedactionLogMergeService {
manualRedactions.getImageRecategorization().forEach(item -> { manualRedactions.getImageRecategorization().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
} }
}); });
manualRedactions.getIdsToRemove().forEach(item -> { manualRedactions.getIdsToRemove().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
} }
}); });
manualRedactions.getForceRedactions().forEach(item -> { manualRedactions.getForceRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
} }
}); });
manualRedactions.getLegalBasisChanges().forEach(item -> { manualRedactions.getLegalBasisChanges().forEach(item -> {
if (item.getSoftDeletedTime() == null) { if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getId().getId(), item.getRequestDate(), item)); manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
} }
}); });
@ -237,10 +230,9 @@ public class RedactionLogMergeService {
for (ManualRedactionEntry manualRedactionEntry : manualAdds) { for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
.getId().getId(), dossierTemplateId);
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions())); redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId().getId())); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
redactionLogEntries.add(redactionLogEntry); redactionLogEntries.add(redactionLogEntry);
} }
} }
@ -248,7 +240,7 @@ public class RedactionLogMergeService {
return redactionLogEntries; return redactionLogEntries;
} }
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Rectangle> positions){ private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions){
return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos return positions.stream().map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos
.getHeight(), pos.getPage())).collect(Collectors.toList()); .getHeight(), pos.getPage())).collect(Collectors.toList());
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.storage; package com.iqser.red.service.redaction.v1.server.storage;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.SectionGrid; import com.iqser.red.service.redaction.v1.model.SectionGrid;
import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.classification.model.Text;

160
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,28 +1,28 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat; import com.amazonaws.services.s3.AmazonS3;
import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
import java.io.BufferedReader; import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import java.io.ByteArrayInputStream; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import java.io.File; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import java.io.FileInputStream; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import java.io.FileOutputStream; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import java.io.IOException; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import java.io.InputStream; import com.iqser.red.service.redaction.v1.model.*;
import java.io.InputStreamReader; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import java.net.URL; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import java.nio.charset.StandardCharsets; import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
import java.time.OffsetDateTime; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import java.util.ArrayList; import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import java.util.HashMap; import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import java.util.HashSet; import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import java.util.List; import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import java.util.Map; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import java.util.Set; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import java.util.UUID; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import java.util.stream.Collectors; import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
@ -47,46 +47,15 @@ import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.junit4.SpringRunner;
import com.amazonaws.services.s3.AmazonS3; import java.io.*;
import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URL;
import com.iqser.red.service.persistence.service.v1.api.model.FileType; import java.nio.charset.StandardCharsets;
import com.iqser.red.service.persistence.service.v1.api.model.JSONPrimitive; import java.time.OffsetDateTime;
import com.iqser.red.service.persistence.service.v1.api.model.ManualRedactions; import java.util.*;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationId; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.data.configuration.Type;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class) @RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -220,7 +189,7 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID)).thenReturn(List.of(Type.builder() when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS +":"+ TEST_DOSSIER_TEMPLATE_ID) .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS) .type(DOSSIER_REDACTIONS)
.dossierTemplateId(TEST_DOSSIER_ID) .dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187") .hexColor("#ffe187")
@ -232,41 +201,41 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(VERTEBRATE, false)); .thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(ADDRESS, false)); .thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(AUTHOR, false)); .thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(SPONSOR, false)); .thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false)); .thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false)); .thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(HINT_ONLY, false)); .thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(MUST_REDACT, false)); .thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false)); .thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(TEST_METHOD, false)); .thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(PII+ ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PII, false)); when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PII, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false)); .thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false)); .thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false)); .thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(PURITY, false)); .thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(IMAGE+ ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMAGE, false)); when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMAGE, false));
when(dictionaryClient.getDictionaryForType(OCR+ ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(OCR, false)); when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(OCR, false));
when(dictionaryClient.getDictionaryForType(LOGO+ ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(LOGO, false)); when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(LOGO, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(SIGNATURE, false)); .thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA+ ":" + TEST_DOSSIER_TEMPLATE_ID)) when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(FORMULA, false)); .thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS+ ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
} }
@ -527,7 +496,7 @@ public class RedactionIntegrationTest {
private Type getDictionaryResponse(String type, boolean isDossierDictionary) { private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
return Type.builder() return Type.builder()
.id(type + ":" +TEST_DOSSIER_TEMPLATE_ID) .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
.hexColor(typeColorMap.get(type)) .hexColor(typeColorMap.get(type))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary
.get(type))) .get(type)))
@ -545,9 +514,8 @@ public class RedactionIntegrationTest {
entries.forEach(entry -> { entries.forEach(entry -> {
dictionaryEntries.add(DictionaryEntry.builder() dictionaryEntries.add(DictionaryEntry.builder()
.value(entry) .value(entry)
.version(reanlysisVersions.containsKey(entry) ? reanlysisVersions.get(entry) : 0L) .version(reanlysisVersions.getOrDefault(entry, 0L))
.deleted(deleted .deleted(deleted.contains(entry)).build());
.contains(entry) ? true : false).build());
}); });
return dictionaryEntries; return dictionaryEntries;
} }
@ -776,7 +744,8 @@ public class RedactionIntegrationTest {
ManualRedactions manualRedactions = new ManualRedactions(); ManualRedactions manualRedactions = new ManualRedactions();
manualRedactions.setImageRecategorization(Set.of(ManualImageRecategorization.builder() manualRedactions.setImageRecategorization(Set.of(ManualImageRecategorization.builder()
.id(new AnnotationId("37eee3e9d589a5cc529bfec38c3ba479", "fileId")) .annotationId("37eee3e9d589a5cc529bfec38c3ba479")
.fileId("fileId")
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
.type("signature") .type("signature")
.build())); .build()));
@ -860,11 +829,13 @@ public class RedactionIntegrationTest {
.text("This is a comment test") .text("This is a comment test")
.build(); .build();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id(new AnnotationId("5b940b2cb401ed9f5be6fc24f6e77bcf", "fileId")) .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.status(AnnotationStatus.DECLINED) .status(AnnotationStatus.DECLINED)
.build())); .build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.id(new AnnotationId("675eba69b0c2917de55462c817adaa05", "fileId")) .annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something") .legalBasis("Something")
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
.build())); .build()));
@ -874,7 +845,8 @@ public class RedactionIntegrationTest {
manualRedactions.getComments().put(manualAddId, List.of(comment)); manualRedactions.getComments().put(manualAddId, List.of(comment));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setId(new AnnotationId(manualAddId, "fileId")); manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED); manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED);
manualRedactionEntry.setTypeId("name:" + TEST_DOSSIER_TEMPLATE_ID); manualRedactionEntry.setTypeId("name:" + TEST_DOSSIER_TEMPLATE_ID);
manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setValue("O'Loughlin C.K.");
@ -904,11 +876,13 @@ public class RedactionIntegrationTest {
manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id(new AnnotationId("5b940b2cb401ed9f5be6fc24f6e77bcf", "fileId")) .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
.build())); .build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.id(new AnnotationId("675eba69b0c2917de55462c817adaa05", "fileId")) .annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change") .legalBasis("Manual Legal Basis Change")
.status(AnnotationStatus.APPROVED) .status(AnnotationStatus.APPROVED)
.build()))); .build())));