Compare commits

...
Author SHA1 Message Date
Dominique Eifländer 012c5a7946 RED-2575: Fixed annotating entries on pages with cropbox != mediabox 2021-10-26 13:51:47 +02:00
Ali Oezyetimoglu a351761257 Pull request #256: RED-2429: As a user I want to resize a redaction
Merge in RED/redaction-service from RED-2429-rs1 to master

* commit 'd8f8702b2a78e7a3bc12e55791c251f3445d5b5e':
  RED-2429: As a user I want to resize a redaction
2021-10-22 12:23:54 +02:00
aoezyetimoglu d8f8702b2a RED-2429: As a user I want to resize a redaction 2021-10-22 12:05:31 +02:00
Dominique Eiflaender 0abcca97b3 Pull request #255: RED-2418: Fixed not working exception handling if redactionlog is not present
Merge in RED/redaction-service from RED-2418 to master

* commit 'f66a054cdea311376ffa88a6641b7eae9be6689f':
  RED-2418: Fixed not working exception handling if redactionlog is not present
2021-10-22 11:05:38 +02:00
Dominique Eifländer f66a054cde RED-2418: Fixed not working exception handling if redactionlog is not present 2021-10-22 10:33:42 +02:00
Dominique Eiflaender 0cc71897b8 Pull request #254: RED-2224: Fixed overriding entries with higher rank
Merge in RED/redaction-service from RED-2224-2 to master

* commit '634eee17167e01e2d4ee4d3d7b9613225e8c5643':
  RED-2224: Fixed overriding entries with higher rank
2021-10-19 15:38:44 +02:00
Dominique Eifländer 634eee1716 RED-2224: Fixed overriding entries with higher rank 2021-10-19 15:32:20 +02:00
Dominique Eiflaender fbad9a759b Pull request #253: RED-2224: Fixed missing values in engines
Merge in RED/redaction-service from RED-2224 to master

* commit 'fa8f598763efb334e21624df468d7e1802477076':
  RED-2224: Fixed missing values in engines
2021-10-19 11:45:38 +02:00
Dominique Eifländer fa8f598763 RED-2224: Fixed missing values in engines 2021-10-19 10:55:51 +02:00
Dominique Eiflaender 46c6385d11 Pull request #250: RED-2223: Fixed adding annotations to rotated words on non rotated pages
Merge in RED/redaction-service from RED-2223 to master

* commit '6005c476962dcdfd1d4e4fabf10fa840377b78db':
  RED-2223: Fixed adding annotations to rotated words on non rotated pages
2021-10-15 15:35:28 +02:00
Dominique Eifländer 6005c47696 RED-2223: Fixed adding annotations to rotated words on non rotated pages 2021-10-15 15:27:15 +02:00
Dominique Eiflaender ac556dfc9f Pull request #249: RED-2468: Fixed push recommendations to dictionary
Merge in RED/redaction-service from RED-2468 to master

* commit '15d836644b6a74273e81de361091d6ea1b71bea9':
  RED-2468: Fixed push recommendations to dictionary
2021-10-15 10:45:10 +02:00
Dominique Eifländer 15d836644b RED-2468: Fixed push recommendations to dictionary 2021-10-15 10:36:01 +02:00
16 changed files with 254 additions and 78 deletions
@@ -20,7 +20,7 @@
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-api-v1</artifactId>
<version>0.44.0</version>
<version>0.77.0</version>
</dependency>
</dependencies>
</project>
@@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -14,4 +16,5 @@ public class AnnotateRequest {
private String dossierId;
private String dossierTemplateId;
private String fileId;
private ManualRedactions manualRedactions;
}
@@ -1,5 +1,5 @@
package com.iqser.red.service.redaction.v1.model;
public enum ManualRedactionType {
ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE
ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE
}
@@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.controller;
import com.iqser.red.commons.spring.ErrorMessage;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
@@ -36,4 +37,11 @@ public class ControllerAdvice {
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
}
@ResponseBody
@ResponseStatus(value = HttpStatus.NOT_FOUND)
@ExceptionHandler(value = NotFoundException.class)
public ErrorMessage handleFileNotFoundException(NotFoundException e) {
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
}
}
@@ -1,11 +1,11 @@
package com.iqser.red.service.redaction.v1.server.controller;
import com.amazonaws.services.kms.model.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
@@ -42,14 +42,14 @@ public class RedactionController implements RedactionResource {
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
var mergedRedactionLog = getRedactionLog(RedactionRequest.builder().fileId(annotateRequest.getFileId()).manualRedactions(annotateRequest.getManualRedactions()).dossierId(annotateRequest.getDossierId()).dossierTemplateId(annotateRequest.getDossierTemplateId()).build());
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
pdDocument.setAllSecurityToBeRemoved(true);
dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
annotationService.annotate(pdDocument, redactionLog, sectionsGrid);
annotationService.annotate(pdDocument, mergedRedactionLog, sectionsGrid);
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
pdDocument.save(byteArrayOutputStream);
@@ -0,0 +1,9 @@
package com.iqser.red.service.redaction.v1.server.exception;
public class NotFoundException extends RuntimeException {
public NotFoundException(String message) {
super(message);
}
}
@@ -21,6 +21,7 @@ public class RedTextPosition {
private float YDirAdj;
private float width;
private float heightDir;
private float widthDirAdj;
// not used in reanalysis
@JsonIgnore
@@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.server.parsing.model;
import static java.util.stream.Collectors.toSet;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.iqser.red.service.redaction.v1.model.Point;
@@ -244,7 +246,15 @@ public class TextPositionSequence implements CharSequence {
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
.getYDirAdj() + height;
} else {
} else if(textPositions.get(0).getRotation() == 0 && textPositions.stream().map(t -> t.getY()).collect(toSet()).size() > 1) {
posYInit = textPositions.get(0).getPageHeight() - getX1();
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
.getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
.getYDirAdj() + height;
}
else {
posXEnd = textPositions.get(textPositions.size() - 1)
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
@@ -65,13 +65,13 @@ public class Dictionary {
public boolean containsValue(String type, String value) {
return localAccessMap.containsKey(type) && localAccessMap.get(type)
.getEntries().stream().map(DictionaryEntry::getValue).collect(toSet())
.getValues(false)
.contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
.getLocalEntries()
.getValues(true)
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
.getEntries().stream().map(DictionaryEntry::getValue).collect(toSet())
.getValues(false)
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
.getLocalEntries()
.getValues(true)
.contains(value);
}
@@ -169,12 +169,12 @@ public class AnnotationService {
private float[] toQuadPoint(Rectangle rectangle, PDRectangle mediaBox, PDRectangle cropBox) {
var x1 = rectangle.getTopLeft().getX() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
var y1 = rectangle.getTopLeft().getY() + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
var x1 = rectangle.getTopLeft().getX() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftX() + (cropBox.toString().equals(mediaBox.toString()) ? cropBox.getLowerLeftX() : 0f);
var y1 = rectangle.getTopLeft().getY() + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY()) + (cropBox.toString().equals(mediaBox.toString()) ? cropBox.getLowerLeftY() : 0f);
var x2 = rectangle.getTopLeft()
.getX() + rectangle.getWidth() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
.getX() + rectangle.getWidth() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftX() + (cropBox.toString().equals(mediaBox.toString()) ? cropBox.getLowerLeftX() : 0f);
var y2 = rectangle.getTopLeft()
.getY() + rectangle.getHeight() - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
.getY() + rectangle.getHeight() - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY()) + (cropBox.toString().equals(mediaBox.toString()) ? cropBox.getLowerLeftY() : 0f);
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
// of the area to be highlighted
@@ -209,18 +209,18 @@ public class EntityRedactionService {
for (DictionaryModel model : dictionary.getDictionaryModels()) {
if (model.isCaseInsensitive()) {
EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), Engine.DICTIONARY));
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
} else {
EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(searchableString, model.getValues(local), model
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), Engine.DICTIONARY));
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
}
}
if (!local) {
Map<String, Set<String>> nerValuesPerType = getNerValues(sectionNumber, nerEntities, cellstarts);
nerValuesPerType.entrySet().forEach(entry -> {
EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(searchableString, entry.getValue(), entry
.getKey(), headline, sectionNumber, false, false, Engine.NER));
EntitySearchUtils.addEntitiesWithHigherRank(found, EntitySearchUtils.find(searchableString, entry.getValue(), entry
.getKey(), headline, sectionNumber, false, false, Engine.NER), dictionary);
});
}
@@ -94,6 +94,12 @@ public class RedactionLogMergeService {
}
});
manualRedactions.getResizeRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
Collections.sort(manualRedactionWrappers);
return manualRedactionWrappers;
@@ -204,6 +210,33 @@ public class RedactionLogMergeService {
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
if (mrw.getItem() instanceof ManualResizeRedaction) {
var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setValue(manualResizeRedact.getValue());
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE);
}
});
}
@@ -146,11 +146,11 @@ public class EntitySearchUtils {
}
var existing = existingOptional.get();
if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
if(existing.getType().equals(found.getType())){
existing.getEngines().addAll(found.getEngines());
} else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){
entities.remove(found);
entities.add(found);
} else {
existing.getEngines().addAll(found.getEngines());
}
} else {
entities.add(found);
@@ -23,7 +23,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
@@ -201,42 +203,26 @@ public class RedactionIntegrationTest {
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
.build()));
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PII, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMAGE, false));
when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(OCR, false));
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(LOGO, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@@ -500,8 +486,7 @@ public class RedactionIntegrationTest {
return Type.builder()
.id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
.hexColor(typeColorMap.get(type))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary
.get(type)))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
.isHint(hintTypeMap.get(type))
.isCaseInsensitive(caseInSensitiveMap.get(type))
.isRecommendation(recommendationTypeMap.get(type))
@@ -517,7 +502,8 @@ public class RedactionIntegrationTest {
dictionaryEntries.add(DictionaryEntry.builder()
.value(entry)
.version(reanlysisVersions.getOrDefault(entry, 0L))
.deleted(deleted.contains(entry)).build());
.deleted(deleted.contains(entry))
.build());
});
return dictionaryEntries;
}
@@ -729,11 +715,9 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
start = System.currentTimeMillis();
@@ -764,14 +748,12 @@ public class RedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument());
}
deleted.remove("mouse");
reanlysisVersions.put("mouse", 4L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
analyzeService.reanalyze(request);
@@ -847,21 +829,19 @@ public class RedactionIntegrationTest {
manualRedactionEntry.setType("name");
manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(Rectangle
.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1).build()
, Rectangle
.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1).build())
);
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1)
.build()));
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
@@ -1064,6 +1044,136 @@ public class RedactionIntegrationTest {
}
@Test
public void resizeRedactionTest() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
ManualRedactions manualRedactions = new ManualRedactions();
String manualAddId = UUID.randomUUID().toString();
Comment comment = Comment.builder()
.date(OffsetDateTime.now())
.user("TEST_USER")
.text("This is a comment test")
.build();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.status(AnnotationStatus.DECLINED)
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
manualRedactions.getComments().put(manualAddId, List.of(comment));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
manualRedactionEntry.setType("CBI_author");
manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1)
.build()));
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
request.setManualRedactions(manualRedactions);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.status(AnnotationStatus.APPROVED)
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build())));
manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder()
.annotationId("fc287b74be2421156ab2895c7474ccdd")
.fileId("fileId")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
.positions(List.of(Rectangle.builder()
.topLeftX(289.44595f)
.topLeftY(327.567f)
.width(7.648041f)
.height(82.51475f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(298.67056f)
.topLeftY(327.567f)
.width(7.648041f)
.height(75.32377f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(307.89517f)
.topLeftY(327.567f)
.width(7.648041f)
.height(61.670967f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(316.99985f)
.topLeftY(327.567f)
.width(7.648041f)
.height(38.104286f)
.page(1)
.build()))
.status(AnnotationStatus.APPROVED)
.build()));
analyzeService.reanalyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.manualRedactions(manualRedactions)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
}
}
private static String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
@@ -1082,7 +1192,9 @@ public class RedactionIntegrationTest {
}
}
private static String getTemporaryDirectory() {
String tmpdir = System.getProperty("java.io.tmpdir");
if (StringUtils.isNotBlank(tmpdir)) {
return tmpdir;