From 2f0e03af34c27cc678d3ea0e99b47258c3e88078 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 4 Mar 2022 11:51:14 +0100 Subject: [PATCH] RED-2836: First steps for false_positive/false_reommendation per entity --- .../annotations/AddRedactionRequest.java | 3 + .../type/DictionaryEntryType.java | 5 + .../api/model/dossiertemplate/type/Type.java | 3 + .../v1/api/resources/DictionaryResource.java | 10 +- .../ManualRedactionEntryEntity.java | 5 + .../configuration/BaseDictionaryEntry.java | 16 ++ .../configuration/DictionaryEntryEntity.java | 2 +- .../DictionaryFalsePositiveEntryEntity.java | 41 ++++ ...tionaryFalseRecommendationEntryEntity.java | 41 ++++ .../entity/configuration/TypeEntity.java | 45 +++- .../DictionaryPersistenceService.java | 5 +- .../persistence/EntryPersistenceService.java | 116 +++++++--- .../AddRedactionPersistenceService.java | 1 + .../FalsePositiveEntryRepository.java | 30 +++ .../FalseRecommendationEntryRepository.java | 31 +++ .../controller/DictionaryController.java | 65 ++++-- .../service/ManualRedactionService.java | 218 +++++++++++------- ...onary_false_positive_tables.changelog.yaml | 111 +++++++++ .../db/changelog/db.changelog-master.yaml | 4 +- .../integration/service/TypeProvider.java | 1 + .../integration/tests/DictionaryTest.java | 95 +++++--- .../tests/DossierTemplateStatsTest.java | 15 +- .../v1/server/integration/tests/TypeTest.java | 9 +- 23 files changed, 671 insertions(+), 201 deletions(-) create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/DictionaryEntryType.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/AddRedactionRequest.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/AddRedactionRequest.java index 925adcb69..76444fd11 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/AddRedactionRequest.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/AddRedactionRequest.java @@ -8,6 +8,8 @@ import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.List; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; + @Data @Builder @AllArgsConstructor @@ -21,6 +23,7 @@ public class AddRedactionRequest { private String legalBasis; private boolean addToDictionary; private boolean addToDossierDictionary; + private DictionaryEntryType dictionaryEntryType; private AnnotationStatus status; private String section; private boolean rectangle; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/DictionaryEntryType.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/DictionaryEntryType.java new file mode 100644 index 000000000..6a54dce2f --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/DictionaryEntryType.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type; + +public enum DictionaryEntryType { + ENTRY, FALSE_POSITIVE, FALSE_RECOMMENDATION +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/Type.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/Type.java index 59e4414ac..881961a23 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/Type.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/Type.java @@ -19,6 +19,7 @@ public class Type { private String type; private String label; private String hexColor; + private String recommendationHexColor; private int rank; private boolean isHint; private boolean isCaseInsensitive; @@ -29,6 +30,8 @@ public class Type { private String dossierTemplateId; private String dossierId; private List entries = new ArrayList<>(); + private List falsePositiveEntries = new ArrayList<>(); + private List falseRecommendationEntries = new ArrayList<>(); // For auto-mappers @JsonIgnore diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java index 775f50cfe..6e95c51c2 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.resources; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -35,13 +36,15 @@ public interface DictionaryResource { void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries, @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent, - @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries); + @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries, + @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType); @ResponseStatus(HttpStatus.NO_CONTENT) @DeleteMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE) void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, - @RequestBody List entries); + @RequestBody List entries, + @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType); @ResponseStatus(HttpStatus.NO_CONTENT) @@ -71,7 +74,8 @@ public interface DictionaryResource { Type getDictionaryForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId); @GetMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE + ENTRIES_PATH, produces = MediaType.APPLICATION_JSON_VALUE) - List getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId); + List getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId, + @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType); @GetMapping(value = VERSION_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java index c81b9893d..57ff599c4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -47,6 +49,9 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation { @Column private boolean addToDossierDictionary; @Column + @Enumerated(EnumType.STRING) + private DictionaryEntryType dictionaryEntryType; + @Column private OffsetDateTime requestDate; @Column private OffsetDateTime processedDate; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java new file mode 100644 index 000000000..d7b1326be --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; + +public interface BaseDictionaryEntry { + + String getValue(); + + + long getVersion(); + + + boolean isDeleted(); + + + TypeEntity getType(); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java index a5964b817..0ef74ed40 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java @@ -14,7 +14,7 @@ import javax.persistence.*; @NoArgsConstructor @AllArgsConstructor @Table(name = "dictionary_entry") -public class DictionaryEntryEntity { +public class DictionaryEntryEntity implements BaseDictionaryEntry { @Id @GeneratedValue diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java new file mode 100644 index 000000000..faf427b30 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@Entity +@NoArgsConstructor +@AllArgsConstructor +@Table(name = "dictionary_false_positive_entry") +public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry { + + @Id + @GeneratedValue + private long entryId; + @Column(length = 4000) + private String value; + @Column + private long version; + @Column + private boolean deleted; + + @JsonIgnore + @ManyToOne(fetch = FetchType.LAZY) + private TypeEntity type; + + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java new file mode 100644 index 000000000..f1d7a760d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@Entity +@NoArgsConstructor +@AllArgsConstructor +@Table(name = "dictionary_false_recommendation_entry") +public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryEntry { + + @Id + @GeneratedValue + private long entryId; + @Column(length = 4000) + private String value; + @Column + private long version; + @Column + private boolean deleted; + + @JsonIgnore + @ManyToOne(fetch = FetchType.LAZY) + private TypeEntity type; + + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java index f350f7fb0..030474ce3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java @@ -1,28 +1,37 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; +import static javax.persistence.CascadeType.ALL; + +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.OneToMany; +import javax.persistence.Table; + import org.hibernate.annotations.BatchSize; import org.hibernate.annotations.Fetch; import org.hibernate.annotations.FetchMode; -import javax.persistence.*; -import java.util.ArrayList; -import java.util.List; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; -import static javax.persistence.CascadeType.ALL; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @Builder @Entity @NoArgsConstructor @AllArgsConstructor -@Table(name = "type") +@Table(name = "entity") public class TypeEntity { @Id @@ -34,6 +43,8 @@ public class TypeEntity { @Column private String hexColor; @Column + private String recommendationHexColor; + @Column private int rank; @Column private boolean isHint; @@ -53,6 +64,16 @@ public class TypeEntity { @OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY) private List entries = new ArrayList<>(); + @Fetch(FetchMode.SELECT) + @BatchSize(size=500) + @OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY) + private List falsePositiveEntries = new ArrayList<>(); + + @Fetch(FetchMode.SELECT) + @BatchSize(size=500) + @OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY) + private List falseRecommendationEntries = new ArrayList<>(); + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "dossier_template_id") private DossierTemplateEntity dossierTemplate; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java index c85d669f7..23058dc7f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java @@ -35,7 +35,7 @@ public class DictionaryPersistenceService { private final EntryRepository entryRepository; - public TypeEntity addType(String type, String dossierTemplateId, String hexColor, int rank, boolean isHint, boolean caseInsensitive, boolean isRecommendation, + public TypeEntity addType(String type, String dossierTemplateId, String hexColor, String recommendationHexColor, int rank, boolean isHint, boolean caseInsensitive, boolean isRecommendation, String description, boolean addToDictionaryAction, String label, String dossierId) { checkRankAlreadyExists(type, dossierTemplateId, rank, dossierId); @@ -46,6 +46,7 @@ public class DictionaryPersistenceService { .dossier(dossierId == null ? null : dossierRepository.getOne(dossierId)) .dossierTemplate(dossierTemplateRepository.getOne(dossierTemplateId)) .hexColor(hexColor) + .recommendationHexColor(recommendationHexColor) .rank(rank) .description(description) .isHint(isHint) @@ -67,7 +68,7 @@ public class DictionaryPersistenceService { typeRepository.findById(typeId).ifPresent((type) -> { type.setVersion(type.getVersion() + 1); checkRankAlreadyExists(type.getType(), type.getDossierTemplate().getId(), typeValueRequest.getRank(), type.getDossier() == null ? null : type.getDossier().getId()); - BeanUtils.copyProperties(typeValueRequest, type, "dossierTemplateId", "dossierId", "entries", "dossierTemplate", "dossier", "id", "version"); + BeanUtils.copyProperties(typeValueRequest, type, "dossierTemplateId", "dossierId", "entries","falsePositiveEntries", "falseRecommendationEntries", "dossierTemplate", "dossier", "id", "version"); }); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java index da846106f..cde51066e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java @@ -1,59 +1,121 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import javax.transaction.Transactional; import java.util.List; import java.util.stream.Collectors; +import javax.transaction.Transactional; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + @Service @Slf4j @RequiredArgsConstructor public class EntryPersistenceService { - private final EntryRepository entryRepository; - private final TypeRepository typeRepository; + private final FalsePositiveEntryRepository falsePositiveEntryRepository; + private final FalseRecommendationEntryRepository falseRecommendationEntryRepository; - public void addEntry(String typeId, List entries, long version) { + public void addEntry(String typeId, List entries, long version, DictionaryEntryType dictionaryEntryType) { var type = typeRepository.getById(typeId); - var dictionaryEntries = entries.stream().map(word -> { - DictionaryEntryEntity entry = new DictionaryEntryEntity(); - entry.setVersion(version); - entry.setValue(word); - entry.setType(type); - return entry; - }).collect(Collectors.toList()); - - entryRepository.saveAll(dictionaryEntries); + switch (dictionaryEntryType) { + case ENTRY: + var dictionaryEntries = entries.stream().map(word -> { + DictionaryEntryEntity entry = new DictionaryEntryEntity(); + entry.setVersion(version); + entry.setValue(word); + entry.setType(type); + return entry; + }).collect(Collectors.toList()); + entryRepository.saveAll(dictionaryEntries); + break; + case FALSE_POSITIVE: + var dictionaryFalsePositiveEntries = entries.stream().map(word -> { + DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity(); + entry.setVersion(version); + entry.setValue(word); + entry.setType(type); + return entry; + }).collect(Collectors.toList()); + falsePositiveEntryRepository.saveAll(dictionaryFalsePositiveEntries); + break; + case FALSE_RECOMMENDATION: + var dictionaryFalseRecommendationsEntries = entries.stream().map(word -> { + DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity(); + entry.setVersion(version); + entry.setValue(word); + entry.setType(type); + return entry; + }).collect(Collectors.toList()); + falseRecommendationEntryRepository.saveAll(dictionaryFalseRecommendationsEntries); + break; + } } @Transactional - public void deleteEntries(String typeId, List values, long version) { - entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values); + public void deleteEntries(String typeId, List values, long version, + DictionaryEntryType dictionaryEntryType) { + + switch (dictionaryEntryType) { + case ENTRY: + entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values); + break; + case FALSE_POSITIVE: + falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values); + break; + case FALSE_RECOMMENDATION: + falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values); + break; + } } @Transactional - public void setVersion(String typeId, List values, long version) { - entryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values); + public void setVersion(String typeId, List values, long version, DictionaryEntryType dictionaryEntryType) { + + switch (dictionaryEntryType) { + case ENTRY: + entryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values); + break; + case FALSE_POSITIVE: + falsePositiveEntryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values); + break; + case FALSE_RECOMMENDATION: + falseRecommendationEntryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values); + break; + } } - public List getEntries(String typeId) { - return entryRepository.findByTypeId(typeId); - + public List getEntries(String typeId, DictionaryEntryType dictionaryEntryType) { + switch (dictionaryEntryType) { + case ENTRY: + return entryRepository.findByTypeId(typeId); + case FALSE_POSITIVE: + return falsePositiveEntryRepository.findByTypeId(typeId); + case FALSE_RECOMMENDATION: + return falseRecommendationEntryRepository.findByTypeId(typeId); + } + return null; } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index 94bd6fe84..b3bd505e4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -39,6 +39,7 @@ public class AddRedactionPersistenceService { manualRedactionEntry.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); manualRedactionEntry.setPositions(convert(addRedactionRequest.getPositions())); manualRedactionEntry.setTypeId(addRedactionRequest.getTypeId()); + manualRedactionEntry.setDictionaryEntryType(addRedactionRequest.getDictionaryEntryType()); if (addRedactionRequest.getStatus() == AnnotationStatus.APPROVED) { manualRedactionEntry.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java new file mode 100644 index 000000000..57c535742 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java @@ -0,0 +1,30 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity; + +public interface FalsePositiveEntryRepository extends JpaRepository { + + @Modifying + @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values") + void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List values); + + + @Modifying + @Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values") + void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List values); + + + @Modifying + @Query("update DictionaryFalsePositiveEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId") + void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId); + + + List findByTypeId(String typeId); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java new file mode 100644 index 000000000..7e583a6dc --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity; + +public interface FalseRecommendationEntryRepository extends JpaRepository { + + @Modifying + @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values") + void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List values); + + + @Modifying + @Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values") + void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List values); + + + @Modifying + @Query("update DictionaryFalseRecommendationEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId") + void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId); + + + List findByTypeId(String typeId); + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java index ae0233712..83fd6a22c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java @@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.controller; import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities; import com.iqser.red.service.peristence.v1.server.service.StopwordService; import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; @@ -13,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource; @@ -56,7 +58,8 @@ public class DictionaryController implements DictionaryResource { @Override public void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries, @RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent, - @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries) { + @RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries, + @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) { Set cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet()); @@ -77,10 +80,10 @@ public class DictionaryController implements DictionaryResource { var currentVersion = getCurrentVersion(typeResult); - List existing = entryPersistenceService.getEntries(typeId) + List existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType) .stream() .filter(e -> !e.isDeleted()) - .map(DictionaryEntryEntity::getValue) + .map(BaseDictionaryEntry::getValue) .collect(toList()); if (removeCurrent) { @@ -91,8 +94,8 @@ public class DictionaryController implements DictionaryResource { List added = new ArrayList<>(cleanEntries); added.removeAll(existing); - entryPersistenceService.deleteEntries(typeId, removed, currentVersion + 1); - entryPersistenceService.addEntry(typeId, added, currentVersion + 1); + entryPersistenceService.deleteEntries(typeId, removed, currentVersion + 1, dictionaryEntryType); + entryPersistenceService.addEntry(typeId, added, currentVersion + 1, dictionaryEntryType); } else { @@ -101,7 +104,7 @@ public class DictionaryController implements DictionaryResource { if (added.isEmpty()) { return; } - entryPersistenceService.addEntry(typeId, added, currentVersion + 1); + entryPersistenceService.addEntry(typeId, added, currentVersion + 1, dictionaryEntryType); } dictionaryPersistenceService.incrementVersion(typeId); @@ -109,7 +112,7 @@ public class DictionaryController implements DictionaryResource { @Override - public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries) { + public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List entries, @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) { // To check whether the type exists Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class); @@ -117,15 +120,15 @@ public class DictionaryController implements DictionaryResource { var currentVersion = getCurrentVersion(typeResult); if (typeResult.isCaseInsensitive()) { - List existing = entryPersistenceService.getEntries(typeId) + List existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY) .stream() - .map(DictionaryEntryEntity::getValue) + .map(BaseDictionaryEntry::getValue) .collect(toList()); entryPersistenceService.deleteEntries(typeId, existing.stream() .filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)) - .collect(toList()), currentVersion + 1); + .collect(toList()), currentVersion + 1, dictionaryEntryType); } else { - entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1); + entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType); } dictionaryPersistenceService.incrementVersion(typeId); @@ -155,11 +158,24 @@ public class DictionaryController implements DictionaryResource { .getRank() != typeValueRequest.getRank()) { var currentVersion = getCurrentVersion(typeResult); - List entries = convert(entryPersistenceService.getEntries(typeId), DictionaryEntry.class); + var entries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY), DictionaryEntry.class); entryPersistenceService.setVersion(typeId, entries.stream() .filter(entry -> !entry.isDeleted()) .map(DictionaryEntry::getValue) - .collect(toList()), currentVersion + 1); + .collect(toList()), currentVersion + 1, DictionaryEntryType.ENTRY); + + var falsePositiveEntries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.FALSE_POSITIVE), DictionaryEntry.class); + entryPersistenceService.setVersion(typeId, falsePositiveEntries.stream() + .filter(entry -> !entry.isDeleted()) + .map(DictionaryEntry::getValue) + .collect(toList()), currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE); + + var falseRecommendationEntries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.FALSE_RECOMMENDATION), DictionaryEntry.class); + entryPersistenceService.setVersion(typeId, falseRecommendationEntries.stream() + .filter(entry -> !entry.isDeleted()) + .map(DictionaryEntry::getValue) + .collect(toList()), currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION); + } } @@ -193,7 +209,7 @@ public class DictionaryController implements DictionaryResource { } String color = typeRequest.getHexColor(); validateColor(color); - return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color, typeRequest + return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color,typeRequest.getRecommendationHexColor(), typeRequest .getRank(), typeRequest.isHint(), typeRequest.isCaseInsensitive(), typeRequest.isRecommendation(), typeRequest .getDescription(), typeRequest.isAddToDictionaryAction(), typeRequest.getLabel(), typeRequest.getDossierId()), Type.class); } @@ -208,12 +224,12 @@ public class DictionaryController implements DictionaryResource { var currentVersion = getCurrentVersion(typeResult); dictionaryPersistenceService.deleteType(typeId); - List existing = entryPersistenceService.getEntries(typeId) + List existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY) .stream() - .map(DictionaryEntryEntity::getValue) + .map(BaseDictionaryEntry::getValue) .collect(toList()); - entryPersistenceService.deleteEntries(typeId, existing, currentVersion + 1); + entryPersistenceService.deleteEntries(typeId, existing, currentVersion + 1, DictionaryEntryType.ENTRY); dictionaryPersistenceService.incrementVersion(typeId); } @@ -239,15 +255,26 @@ public class DictionaryController implements DictionaryResource { var entity = dictionaryPersistenceService.getType(typeId); var target = convert(entity, Type.class); target.setEntries(convert(entity.getEntries(), DictionaryEntry.class)); + target.setFalsePositiveEntries(convert(entity.getEntries(), DictionaryEntry.class)); + target.setFalseRecommendationEntries(convert(entity.getEntries(), DictionaryEntry.class)); return target; } @Override @Transactional - public List getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId) { + public List getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId, + @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) { - return convert(dictionaryPersistenceService.getType(typeId).getEntries(), DictionaryEntry.class); + switch (dictionaryEntryType) { + case ENTRY: + return convert(dictionaryPersistenceService.getType(typeId).getEntries(), DictionaryEntry.class); + case FALSE_POSITIVE: + return convert(dictionaryPersistenceService.getType(typeId).getFalsePositiveEntries(), DictionaryEntry.class); + case FALSE_RECOMMENDATION: + return convert(dictionaryPersistenceService.getType(typeId).getFalseRecommendationEntries(), DictionaryEntry.class); + } + return null; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 10c7a3d6b..471c94896 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -60,6 +60,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Resize import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.MessageType; import com.iqser.red.service.redaction.v1.model.RedactionLog; @@ -119,12 +120,12 @@ public class ManualRedactionService { commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId(); } - var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId); + var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType()); - if(actionPerformed){ + if (actionPerformed) { // in case of add to dict, there is no need to process surrounding text - reprocess(dossierId,fileId); - }else{ + reprocess(dossierId, fileId); + } else { if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest.isRectangle()) { var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper()); ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build(); @@ -138,14 +139,15 @@ public class ManualRedactionService { } - public List addRemoveRedaction(String dossierId, String fileId, List removeRedactionRequests) { + public List addRemoveRedaction(String dossierId, String fileId, + List removeRedactionRequests) { var response = new ArrayList(); var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var actionPerformed = false; var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId); - for(var removeRedactionRequest : removeRedactionRequests ){ + for (var removeRedactionRequest : removeRedactionRequests) { removeRedactionPersistenceService.insert(fileId, removeRedactionRequest); @@ -156,35 +158,10 @@ public class ManualRedactionService { actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog, dossier, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), false); - response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build()); - } - - if(actionPerformed){ - reprocess(dossierId, fileId); - } - - analysisFlagsCalculationService.calculateFlags(dossierId, fileId); - - return response; - } - - - public List addForceRedaction(String dossierId, String fileId, List forceRedactionRequests) { - - var response = new ArrayList(); - dossierPersistenceService.getAndValidateDossier(dossierId); - var actionPerformed = false; - - for(var forceRedactionRequest : forceRedactionRequests ) { - forceRedactionPersistenceService.insert(fileId, forceRedactionRequest); - - Long commentId = null; - if (forceRedactionRequest.getComment() != null) { - - commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId(); - } - actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus()); - response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build()); + response.add(ManualAddResponse.builder() + .annotationId(removeRedactionRequest.getAnnotationId()) + .commentId(commentId) + .build()); } if (actionPerformed) { @@ -197,12 +174,45 @@ public class ManualRedactionService { } - public List addLegalBasisChange(String dossierId, String fileId, List legalBasisChangeRequests) { + public List addForceRedaction(String dossierId, String fileId, + List forceRedactionRequests) { + + var response = new ArrayList(); + dossierPersistenceService.getAndValidateDossier(dossierId); + var actionPerformed = false; + + for (var forceRedactionRequest : forceRedactionRequests) { + forceRedactionPersistenceService.insert(fileId, forceRedactionRequest); + + Long commentId = null; + if (forceRedactionRequest.getComment() != null) { + + commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId(); + } + actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus()); + response.add(ManualAddResponse.builder() + .annotationId(forceRedactionRequest.getAnnotationId()) + .commentId(commentId) + .build()); + } + + if (actionPerformed) { + reprocess(dossierId, fileId); + } + + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); + + return response; + } + + + public List addLegalBasisChange(String dossierId, String fileId, + List legalBasisChangeRequests) { var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); - for(var legalBasisChangeRequest : legalBasisChangeRequests) { + for (var legalBasisChangeRequest : legalBasisChangeRequests) { legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest); Long commentId = null; @@ -211,7 +221,10 @@ public class ManualRedactionService { commentId = addComment(fileId, legalBasisChangeRequest.getAnnotationId(), legalBasisChangeRequest.getComment(), legalBasisChangeRequest.getUser()).getId(); } - response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build()); + response.add(ManualAddResponse.builder() + .annotationId(legalBasisChangeRequest.getAnnotationId()) + .commentId(commentId) + .build()); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); @@ -219,13 +232,14 @@ public class ManualRedactionService { } - public List addImageRecategorization(String dossierId, String fileId, List imageRecategorizationRequests) { + public List addImageRecategorization(String dossierId, String fileId, + List imageRecategorizationRequests) { var response = new ArrayList(); var actionPerformed = false; dossierPersistenceService.getAndValidateDossier(dossierId); - for(var imageRecategorizationRequest : imageRecategorizationRequests) { + for (var imageRecategorizationRequest : imageRecategorizationRequests) { recategorizationPersistenceService.insert(fileId, imageRecategorizationRequest); Long commentId = null; @@ -236,7 +250,10 @@ public class ManualRedactionService { actionPerformed = actionPerformed || !AnnotationStatus.REQUESTED.equals(imageRecategorizationRequest.getStatus()); - response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build()); + response.add(ManualAddResponse.builder() + .annotationId(imageRecategorizationRequest.getAnnotationId()) + .commentId(commentId) + .build()); } if (actionPerformed) { reprocess(dossierId, fileId); @@ -306,9 +323,9 @@ public class ManualRedactionService { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var actionPerformed = false; - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { var addRedaction = getAddRedaction(fileId, annotationId); - actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedaction.getTypeId(), addRedaction.getValue(), addRedaction.getStatus(), addRedaction.isAddToDictionary(), addRedaction.isAddToDossierDictionary(), true, dossier.getId()); + actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedaction.getTypeId(), addRedaction.getValue(), addRedaction.getStatus(), addRedaction.isAddToDictionary(), addRedaction.isAddToDossierDictionary(), true, dossier.getId(), addRedaction.getDictionaryEntryType()); addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); } @@ -326,15 +343,15 @@ public class ManualRedactionService { var actionPerformed = false; var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId); - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { var removeRedaction = getRemoveRedaction(fileId, annotationId); - actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog , dossier, fileId, annotationId, removeRedaction.getStatus(), removeRedaction.isRemoveFromDictionary(), true); + actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog, dossier, fileId, annotationId, removeRedaction.getStatus(), removeRedaction.isRemoveFromDictionary(), true); removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); } - if(actionPerformed) { + if (actionPerformed) { reprocess(dossierId, fileId); } @@ -346,7 +363,7 @@ public class ManualRedactionService { var now = OffsetDateTime.now(); - for(var annotationId : annotationIds ) { + for (var annotationId : annotationIds) { forceRedactionPersistenceService.softDelete(fileId, annotationId, now); } @@ -356,7 +373,7 @@ public class ManualRedactionService { public void deleteLegalBasisChange(String dossierId, String fileId, List annotationIds) { - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { legalBasisChangePersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); } @@ -367,7 +384,7 @@ public class ManualRedactionService { public void deleteImageRecategorization(String dossierId, String fileId, List annotationIds) { var actionPerformed = false; - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { var imageRecategorization = getImageRecategorization(fileId, annotationId); recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now()); @@ -383,8 +400,9 @@ public class ManualRedactionService { public void deleteComment(String fileId, List commentIds) { + fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); - for(var commentId : commentIds) { + for (var commentId : commentIds) { commentPersistenceService.softDelete(commentId, OffsetDateTime.now()); } // update indicator @@ -395,19 +413,20 @@ public class ManualRedactionService { public void deleteResizeRedaction(String dossierId, String fileId, List annotationIds) { OffsetDateTime now = OffsetDateTime.now(); - for(var annotationId : annotationIds ) { + for (var annotationId : annotationIds) { resizeRedactionPersistenceService.softDelete(fileId, annotationId, now); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } - public List addResizeRedaction(String dossierId, String fileId, List resizeRedactionRequests) { + public List addResizeRedaction(String dossierId, String fileId, + List resizeRedactionRequests) { var response = new ArrayList(); dossierPersistenceService.getAndValidateDossier(dossierId); - for(var resizeRedactionRequest : resizeRedactionRequests ) { + for (var resizeRedactionRequest : resizeRedactionRequests) { resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest); Long commentId = null; @@ -419,7 +438,10 @@ public class ManualRedactionService { ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build(); addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions); - response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build()); + response.add(ManualAddResponse.builder() + .annotationId(resizeRedactionRequest.getAnnotationId()) + .commentId(commentId) + .build()); } analysisFlagsCalculationService.calculateFlags(dossierId, fileId); @@ -429,16 +451,16 @@ public class ManualRedactionService { @SuppressWarnings("PMD") - public void updateRemoveRedactionStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { + public void updateRemoveRedactionStatus(String dossierId, String fileId, List annotationIds, + AnnotationStatus annotationStatus) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId); - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { IdRemovalEntity idRemoval = removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId); if (idRemoval.isRemoveFromDictionary()) { - Optional redactionLogEntryOptional = redactionLog.getRedactionLogEntry() .stream() .filter(entry -> entry.getId().equals(idRemoval.getId().getAnnotationId())) @@ -451,14 +473,14 @@ public class ManualRedactionService { var redactionLogEntry = redactionLogEntryOptional.get(); if (annotationStatus == AnnotationStatus.APPROVED) { - removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId); + removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY); approveStatusForRedactionsWithSameValue(dossier, false, true, redactionLogEntry.getValue()); } else if (annotationStatus == AnnotationStatus.DECLINED) { // if it was previously approved, revert the delete if (idRemoval.getStatus() == AnnotationStatus.APPROVED) { - addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId); + addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY); } } } @@ -472,10 +494,11 @@ public class ManualRedactionService { } - public void updateForceRedactionStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { + public void updateForceRedactionStatus(String dossierId, String fileId, List annotationIds, + AnnotationStatus annotationStatus) { dossierPersistenceService.getAndValidateDossier(dossierId); - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus); // boolean hasSuggestions = calculateHasSuggestions(fileId); // fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions); @@ -491,10 +514,11 @@ public class ManualRedactionService { @Transactional - public void updateLegalBasisChangeStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { + public void updateLegalBasisChangeStatus(String dossierId, String fileId, List annotationIds, + AnnotationStatus annotationStatus) { dossierPersistenceService.getAndValidateDossier(dossierId); - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus); // boolean hasSuggestions = calculateHasSuggestions(fileId); // fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions); @@ -503,16 +527,16 @@ public class ManualRedactionService { } - public void updateImageRecategorizationStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { + public void updateImageRecategorizationStatus(String dossierId, String fileId, List annotationIds, + AnnotationStatus annotationStatus) { var actionPerformed = false; dossierPersistenceService.getAndValidateDossier(dossierId); - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { ManualImageRecategorizationEntity imageRecategorization = recategorizationPersistenceService.findRecategorization(fileId, annotationId); - actionPerformed = actionPerformed || - AnnotationStatus.DECLINED.equals(annotationStatus) && AnnotationStatus.APPROVED.equals(imageRecategorization.getStatus()); + actionPerformed = actionPerformed || AnnotationStatus.DECLINED.equals(annotationStatus) && AnnotationStatus.APPROVED.equals(imageRecategorization.getStatus()); actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(annotationStatus); recategorizationPersistenceService.updateStatus(fileId, annotationId, annotationStatus); @@ -537,7 +561,8 @@ public class ManualRedactionService { } - public void updateResizeRedactionStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { + public void updateResizeRedactionStatus(String dossierId, String fileId, List annotationIds, + AnnotationStatus annotationStatus) { dossierPersistenceService.getAndValidateDossier(dossierId); for (var annotationId : annotationIds) { @@ -550,26 +575,28 @@ public class ManualRedactionService { @SuppressWarnings("PMD") - public void updateAddRedactionStatus(String dossierId, String fileId, List annotationIds, AnnotationStatus annotationStatus) { + public void updateAddRedactionStatus(String dossierId, String fileId, List annotationIds, + AnnotationStatus annotationStatus) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); - for(var annotationId : annotationIds) { + for (var annotationId : annotationIds) { ManualRedactionEntryEntity manualRedactionEntry = addRedactionPersistenceService.findAddRedaction(fileId, annotationId); if (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) { if (annotationStatus == AnnotationStatus.APPROVED) { - addToDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId); + addToDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId, manualRedactionEntry.getDictionaryEntryType()); approveStatusForRedactionsWithSameValue(dossier, manualRedactionEntry.isAddToDictionary(), manualRedactionEntry.isAddToDossierDictionary(), manualRedactionEntry.getValue()); } else if (annotationStatus == AnnotationStatus.DECLINED) { // if it was previously approved, revert the add if (manualRedactionEntry.getStatus() == AnnotationStatus.APPROVED) { - removeFromDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId); + removeFromDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId, manualRedactionEntry.getDictionaryEntryType()); } } } - addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS)); // boolean hasSuggestions = calculateHasSuggestions(fileId); // fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasSuggestions); @@ -591,7 +618,8 @@ public class ManualRedactionService { manualRedactions.getForceRedactions().forEach(forceRedactionPersistenceService::markAsProcessed); } if (manualRedactions.getImageRecategorization() != null) { - manualRedactions.getImageRecategorization().forEach(recategorizationPersistenceService::markAsProcessed); + manualRedactions.getImageRecategorization() + .forEach(recategorizationPersistenceService::markAsProcessed); } } } @@ -601,7 +629,12 @@ public class ManualRedactionService { fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING); - var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build(); + var analyseRequest = AnalyzeRequest.builder() + .messageType(MessageType.SURROUNDING_TEXT) + .dossierId(dossierId) + .fileId(fileId) + .manualRedactions(manualRedactions) + .build(); try { rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> { @@ -615,11 +648,13 @@ public class ManualRedactionService { } - private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary, boolean addToDossierDictionary, String value) { + private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary, + boolean addToDossierDictionary, String value) { List dossiers = new ArrayList<>(); if (addToDictionary) { - dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId()).getDossiers(); + dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId()) + .getDossiers(); } if (addToDossierDictionary) { @@ -645,13 +680,14 @@ public class ManualRedactionService { } - private boolean handleRemoveFromDictionary(RedactionLog redactionLog, DossierEntity dossier, String fileId, String annotationId, AnnotationStatus status, boolean removeFromDictionary, boolean revert) { + private boolean handleRemoveFromDictionary(RedactionLog redactionLog, DossierEntity dossier, String fileId, + String annotationId, AnnotationStatus status, + boolean removeFromDictionary, boolean revert) { if (status == AnnotationStatus.APPROVED) { if (removeFromDictionary) { - Optional redactionLogEntryOptional = redactionLog.getRedactionLogEntry() .stream() .filter(entry -> entry.getId().equals(annotationId)) @@ -664,9 +700,9 @@ public class ManualRedactionService { var redactionLogEntry = redactionLogEntryOptional.get(); if (revert) { - addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId); + addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY); } else { - removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId); + removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY); } return true; @@ -677,22 +713,24 @@ public class ManualRedactionService { } - private void addToDictionary(String typeId, String value, String dossierId, String fileId) { + private void addToDictionary(String typeId, String value, String dossierId, String fileId, + DictionaryEntryType dictionaryEntryType) { try { log.debug("Adding entry: {} to {} for {} / {}", value, typeId, dossierId, fileId); - dictionaryController.addEntries(typeId, List.of(value), false, false); + dictionaryController.addEntries(typeId, List.of(value), false, false, dictionaryEntryType != null ? dictionaryEntryType : DictionaryEntryType.ENTRY); } catch (Exception e) { throw new BadRequestException(e.getMessage()); } } - private void removeFromDictionary(String typeId, String value, String dossierId, String fileId) { + private void removeFromDictionary(String typeId, String value, String dossierId, String fileId, + DictionaryEntryType dictionaryEntryType) { try { log.debug("Deleting entries to {} for {} / {}", typeId, dossierId, fileId); - dictionaryController.deleteEntries(typeId, List.of(value)); + dictionaryController.deleteEntries(typeId, List.of(value), dictionaryEntryType != null ? dictionaryEntryType : DictionaryEntryType.ENTRY); } catch (FeignException e) { throw new BadRequestException(e.getMessage()); } @@ -711,17 +749,19 @@ public class ManualRedactionService { } - private boolean handleAddToDictionary(String fileId, String annotationId, String typeId, String value, AnnotationStatus status, boolean addToDictionary, - boolean addToDossierDictionary, boolean revert, String dossierId) { + private boolean handleAddToDictionary(String fileId, String annotationId, String typeId, String value, + AnnotationStatus status, boolean addToDictionary, + boolean addToDossierDictionary, boolean revert, String dossierId, + DictionaryEntryType dictionaryEntryType) { if (status == AnnotationStatus.APPROVED) { addRedactionPersistenceService.updateStatus(fileId, annotationId, status, addToDictionary, addToDossierDictionary); if (addToDictionary || addToDossierDictionary) { if (revert) { - removeFromDictionary(typeId, value, dossierId, fileId); + removeFromDictionary(typeId, value, dossierId, fileId, dictionaryEntryType); } else { - addToDictionary(typeId, value, dossierId, fileId); + addToDictionary(typeId, value, dossierId, fileId, dictionaryEntryType); } return true; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml new file mode 100644 index 000000000..56da81c0a --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml @@ -0,0 +1,111 @@ +databaseChangeLog: + - changeSet: + id: add-recommendaction-color + author: dom + changes: + - addColumn: + columns: + - column: + name: recommendation_hex_color + type: VARCHAR(255) + tableName: type + - changeSet: + id: 11-added-dictionary_false_positive_table + author: dom + changes: + - createTable: + columns: + - column: + constraints: + nullable: false + primaryKey: true + primaryKeyName: dictionary_false_positive_entry_pkey + name: entry_id + type: BIGINT + - column: + name: deleted + type: BOOLEAN + - column: + name: value + type: VARCHAR(4000) + - column: + name: version + type: BIGINT + - column: + name: type_id + type: VARCHAR(255) + tableName: dictionary_false_positive_entry + - changeSet: + id: 11-added-dictionary_false_positive_table-type-fk + author: dom + changes: + - addForeignKeyConstraint: + baseColumnNames: type_id + baseTableName: dictionary_false_positive_entry + constraintName: dictionary_false_positive_table-type-fk-constraint + deferrable: false + initiallyDeferred: false + onDelete: NO ACTION + onUpdate: NO ACTION + referencedColumnNames: id + referencedTableName: type + validate: true + - changeSet: + id: 11-dictionary_false_recommendation_entry_table + author: dom + changes: + - createTable: + columns: + - column: + constraints: + nullable: false + primaryKey: true + primaryKeyName: dictionary_false_recommendation_entry_pkey + name: entry_id + type: BIGINT + - column: + name: deleted + type: BOOLEAN + - column: + name: value + type: VARCHAR(4000) + - column: + name: version + type: BIGINT + - column: + name: type_id + type: VARCHAR(255) + tableName: dictionary_false_recommendation_entry + - changeSet: + id: 11-dictionary_false_recommendation_entry_table-type-fk + author: dom + changes: + - addForeignKeyConstraint: + baseColumnNames: type_id + baseTableName: dictionary_false_recommendation_entry + constraintName: dictionary_false_recommendation_entry_table-type-fk-constraint + deferrable: false + initiallyDeferred: false + onDelete: NO ACTION + onUpdate: NO ACTION + referencedColumnNames: id + referencedTableName: type + validate: true + - changeSet: + id: rename-type-table-to-entity + author: dom + changes: + - renameTable: + oldTableName: type + newTableName: entity + - changeSet: + id: added-dictionaryEntryType-to-manual-redaction + author: dom + changes: + - addColumn: + columns: + - column: + name: dictionary_entry_type + type: VARCHAR(255) + defaultValue: ENTRY + tableName: manual_redaction \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 625b12b38..9465ec09a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -22,4 +22,6 @@ databaseChangeLog: - include: file: db/changelog/10-added-file-manipulation-date.changelog.yaml - include: - file: db/changelog/sql/10-set-file-manipulation-date.sql \ No newline at end of file + file: db/changelog/sql/10-set-file-manipulation-date.sql + - include: + file: db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java index 3b3cd36f1..36281065a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java @@ -23,6 +23,7 @@ public class TypeProvider { type.setDescription("test"); type.setAddToDictionaryAction(true); type.setHexColor("#dddddd"); + type.setRecommendationHexColor("#aaaaaa"); type.setHint(false); type.setRank(100); type.setRecommendation(false); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java index 3c50f0bd5..c4c9ce440 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java @@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; import org.junit.Test; @@ -14,6 +15,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import feign.FeignException; @@ -36,6 +38,27 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { private DossierTemplateClient dossierTemplateClient; + @Test + public void testAddFalsePositiveAndFalseRecommendation() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + var type = typeProvider.testAndProvideType(dossierTemplate); + assertThat(type.getRecommendationHexColor()).isEqualTo("#aaaaaa"); + + dictionaryClient.addEntries(type.getTypeId(), List.of("word1", "word2"), false, false, DictionaryEntryType.ENTRY); + dictionaryClient.addEntries(type.getTypeId(), List.of("false_positive1", "false_positive"), false, false, DictionaryEntryType.FALSE_POSITIVE); + dictionaryClient.addEntries(type.getTypeId(), List.of("false_recommendation1", "false_recommendation2"), false, false, DictionaryEntryType.FALSE_RECOMMENDATION); + + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(2); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.FALSE_POSITIVE)).hasSize(2); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.FALSE_RECOMMENDATION)).hasSize(2); + + var loadedType = dictionaryClient.getDictionaryForType(type.getTypeId()); + assertThat(loadedType.getEntries()).hasSize(2); + assertThat(loadedType.getFalsePositiveEntries()).hasSize(2); + assertThat(loadedType.getFalseRecommendationEntries()).hasSize(2); + } + @Test public void testAddEntriesWithStopWord() { @@ -45,7 +68,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add("age"); entries.add("page"); try { - dictionaryClient.addEntries(typeId, entries, false, false); + dictionaryClient.addEntries(typeId, entries, false, false, DictionaryEntryType.ENTRY); } catch (FeignException e) { assertThat(e.status()).isEqualTo(400); } @@ -64,18 +87,18 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { var entries = new ArrayList(); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, false, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); // Act & Assert: Add same word again; Only one should exist entries = new ArrayList<>(); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, false, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); // Act & Assert: Add same word multiple times again; Only one should exist entries = new ArrayList<>(); @@ -83,19 +106,19 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, false, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); // Act & Assert: Delete word; Should have 'deleted' flag entries = new ArrayList<>(); entries.add(word); - dictionaryClient.deleteEntries(type.getTypeId(), entries); + dictionaryClient.deleteEntries(type.getTypeId(), entries, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).isDeleted()).isTrue(); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).isDeleted()).isTrue(); } @@ -111,18 +134,18 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { var entries = new ArrayList(); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, true, false); + dictionaryClient.addEntries(type.getTypeId(), entries, true, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); // Act & Assert: Add same word again; Only one should exist entries = new ArrayList<>(); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, true, false); + dictionaryClient.addEntries(type.getTypeId(), entries, true, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); // Act & Assert: Add same word multiple times again; Only one should exist entries = new ArrayList<>(); @@ -130,19 +153,19 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word); entries.add(word); entries.add(word); - dictionaryClient.addEntries(type.getTypeId(), entries, true, false); + dictionaryClient.addEntries(type.getTypeId(), entries, true, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); // Act & Assert: Delete word; Should have 'deleted' flag entries = new ArrayList<>(); entries.add(word); - dictionaryClient.deleteEntries(type.getTypeId(), entries); + dictionaryClient.deleteEntries(type.getTypeId(), entries, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).isDeleted()).isTrue(); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).isDeleted()).isTrue(); } @@ -161,10 +184,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word1); entries.add(word2); entries.add(word3); - dictionaryClient.addEntries(type.getTypeId(), entries, false, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()) + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(3); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY) .stream() .map(e -> e.getValue()) .collect(Collectors.toList())).contains(word1, word2, word3); @@ -172,10 +195,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { // Act & Assert: Add same word again; No duplicate should exist entries = new ArrayList<>(); entries.add(word1); - dictionaryClient.addEntries(type.getTypeId(), entries, false, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()) + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(3); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY) .stream() .map(e -> e.getValue()) .collect(Collectors.toList())).contains(word1, word2, word3); @@ -188,10 +211,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest { entries.add(word1); entries.add(word2); entries.add(word3); - dictionaryClient.addEntries(type.getTypeId(), entries, false, false); + dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3); - assertThat(dictionaryClient.getEntriesForType(type.getTypeId()) + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(3); + assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY) .stream() .map(e -> e.getValue()) .collect(Collectors.toList())).contains(word1, word2, word3); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 68e1ea16a..2fd07519f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -11,6 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.Do import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import org.junit.Test; @@ -77,9 +78,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var entries1 = new ArrayList(); entries1.add("entry1"); entries1.add("entry2"); - dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false); + dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false, DictionaryEntryType.ENTRY); - List entryList = dictionaryClient.getEntriesForType(addedType.getTypeId()); + List entryList = dictionaryClient.getEntriesForType(addedType.getTypeId(), DictionaryEntryType.ENTRY); assertThat(entryList.size()).isEqualTo(entries1.size()); var addedType2 = dictionaryClient.addType(Type.builder() @@ -96,9 +97,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe entries2.add("entry1"); entries2.add("entry2"); entries2.add("entry3"); - dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false); + dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false, DictionaryEntryType.ENTRY); - entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId()); + entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId(), DictionaryEntryType.ENTRY); assertThat(entryList.size()).isEqualTo(entries2.size()); var dossierTemplate3 = provideTestTemplate("dossierTemp3"); @@ -117,9 +118,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var entries3 = new ArrayList(); entries3.add("entry1"); entries3.add("entry2"); - dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false); + dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false, DictionaryEntryType.ENTRY); - entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId()); + entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId(), DictionaryEntryType.ENTRY); assertThat(entryList.size()).isEqualTo(entries3.size()); var dossierTemplate4 = provideTestTemplate("dossierTemp4"); @@ -149,7 +150,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var entries22 = new ArrayList(); entries22.add(entries2.get(2)); - dictionaryClient.deleteEntries(addedType2.getTypeId(), entries22); + dictionaryClient.deleteEntries(addedType2.getTypeId(), entries22, DictionaryEntryType.ENTRY); dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateStats(dossierTemplateIds); assertThat(dossierTemplateStatsList.size()).isEqualTo(dossierTemplateIds.size()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java index cdf28a843..9de762e07 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java @@ -6,6 +6,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import org.assertj.core.util.Lists; import org.junit.Test; @@ -35,7 +36,7 @@ public class TypeTest extends AbstractPersistenceServerServiceTest { var type = typeProvider.testAndProvideType(dossierTemplate); - dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false); + dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false, DictionaryEntryType.ENTRY); var loadedType = dictionaryClient.getDictionaryForType(type.getId()); @@ -43,17 +44,17 @@ public class TypeTest extends AbstractPersistenceServerServiceTest { assertThat(loadedType.getVersion()).isGreaterThan(type.getVersion()); - var dict = dictionaryClient.getEntriesForType(type.getId()); + var dict = dictionaryClient.getEntriesForType(type.getId(), DictionaryEntryType.ENTRY); assertThat(dict.size()).isEqualTo(3); - dictionaryClient.deleteEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc")); + dictionaryClient.deleteEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), DictionaryEntryType.ENTRY); loadedType = dictionaryClient.getDictionaryForType(type.getId()); assertThat(loadedType.getVersion()).isGreaterThan(type.getVersion() + 1); - dict = dictionaryClient.getEntriesForType(type.getId()); + dict = dictionaryClient.getEntriesForType(type.getId(), DictionaryEntryType.ENTRY); assertThat(dict.size()).isEqualTo(3); for (var entry : dict) {