Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #272: RED-2836: First steps for false_positive/false_reommendation per entity

Browse Source 
Merge in RED/persistence-service from RED-2836 to master

* commit '2f0e03af34c27cc678d3ea0e99b47258c3e88078':
  RED-2836: First steps for false_positive/false_reommendation per entity
This commit is contained in:
Dominique Eiflaender 2022-03-04 13:19:52 +01:00
commit 1c4012d471
23 changed files with 671 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/AddRedactionRequest.java
View File

@ -8,6 +8,8 @@ import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
@Data
@Builder
@AllArgsConstructor
@ -21,6 +23,7 @@ public class AddRedactionRequest {
private String legalBasis;
private boolean addToDictionary;
private boolean addToDossierDictionary;
private DictionaryEntryType dictionaryEntryType;
private AnnotationStatus status;
private String section;
private boolean rectangle;

5
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/DictionaryEntryType.java Normal file
View File

@ -0,0 +1,5 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type;
public enum DictionaryEntryType {
ENTRY, FALSE_POSITIVE, FALSE_RECOMMENDATION
}

3
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/type/Type.java
View File

@ -19,6 +19,7 @@ public class Type {
private String type;
private String label;
private String hexColor;
private String recommendationHexColor;
private int rank;
private boolean isHint;
private boolean isCaseInsensitive;
@ -29,6 +30,8 @@ public class Type {
private String dossierTemplateId;
private String dossierId;
private List<DictionaryEntry> entries = new ArrayList<>();
private List<DictionaryEntry> falsePositiveEntries = new ArrayList<>();
private List<DictionaryEntry> falseRecommendationEntries = new ArrayList<>();
// For auto-mappers
@JsonIgnore

10
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DictionaryResource.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.resources;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
@ -35,13 +36,15 @@ public interface DictionaryResource {
void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId,
@RequestBody List<String> entries,
@RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent,
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries);
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries,
@RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType);
@ResponseStatus(HttpStatus.NO_CONTENT)
@DeleteMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId,
@RequestBody List<String> entries);
@RequestBody List<String> entries,
@RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType);
@ResponseStatus(HttpStatus.NO_CONTENT)
@ -71,7 +74,8 @@ public interface DictionaryResource {
Type getDictionaryForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId);
@GetMapping(value = DICTIONARY_PATH + TYPE_PATH_VARIABLE + ENTRIES_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
List<DictionaryEntry> getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId);
List<DictionaryEntry> getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId,
@RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType);
@GetMapping(value = VERSION_PATH + DOSSIER_TEMPLATE_PATH_VARIABLE)

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java
View File

@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.annotat
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -47,6 +49,9 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation {
@Column
private boolean addToDossierDictionary;
@Column
@Enumerated(EnumType.STRING)
private DictionaryEntryType dictionaryEntryType;
@Column
private OffsetDateTime requestDate;
@Column
private OffsetDateTime processedDate;

16
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java Normal file
View File

@ -0,0 +1,16 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
public interface BaseDictionaryEntry {
String getValue();
long getVersion();
boolean isDeleted();
TypeEntity getType();
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java
View File

@ -14,7 +14,7 @@ import javax.persistence.*;
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "dictionary_entry")
public class DictionaryEntryEntity {
public class DictionaryEntryEntity implements BaseDictionaryEntry {
@Id
@GeneratedValue

41
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "dictionary_false_positive_entry")
public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry {
@Id
@GeneratedValue
private long entryId;
@Column(length = 4000)
private String value;
@Column
private long version;
@Column
private boolean deleted;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
private TypeEntity type;
}

41
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "dictionary_false_recommendation_entry")
public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryEntry {
@Id
@GeneratedValue
private long entryId;
@Column(length = 4000)
private String value;
@Column
private long version;
@Column
private boolean deleted;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
private TypeEntity type;
}

45
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java
View File

@ -1,28 +1,37 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import static javax.persistence.CascadeType.ALL;
import java.util.ArrayList;
import java.util.List;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import org.hibernate.annotations.BatchSize;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import static javax.persistence.CascadeType.ALL;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "type")
@Table(name = "entity")
public class TypeEntity {
@Id
@ -34,6 +43,8 @@ public class TypeEntity {
@Column
private String hexColor;
@Column
private String recommendationHexColor;
@Column
private int rank;
@Column
private boolean isHint;
@ -53,6 +64,16 @@ public class TypeEntity {
@OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY)
private List<DictionaryEntryEntity> entries = new ArrayList<>();
@Fetch(FetchMode.SELECT)
@BatchSize(size=500)
@OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY)
private List<DictionaryFalsePositiveEntryEntity> falsePositiveEntries = new ArrayList<>();
@Fetch(FetchMode.SELECT)
@BatchSize(size=500)
@OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY)
private List<DictionaryFalseRecommendationEntryEntity> falseRecommendationEntries = new ArrayList<>();
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "dossier_template_id")
private DossierTemplateEntity dossierTemplate;

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java
View File

@ -35,7 +35,7 @@ public class DictionaryPersistenceService {
private final EntryRepository entryRepository;
public TypeEntity addType(String type, String dossierTemplateId, String hexColor, int rank, boolean isHint, boolean caseInsensitive, boolean isRecommendation,
public TypeEntity addType(String type, String dossierTemplateId, String hexColor, String recommendationHexColor, int rank, boolean isHint, boolean caseInsensitive, boolean isRecommendation,
String description, boolean addToDictionaryAction, String label, String dossierId) {
checkRankAlreadyExists(type, dossierTemplateId, rank, dossierId);
@ -46,6 +46,7 @@ public class DictionaryPersistenceService {
.dossier(dossierId == null ? null : dossierRepository.getOne(dossierId))
.dossierTemplate(dossierTemplateRepository.getOne(dossierTemplateId))
.hexColor(hexColor)
.recommendationHexColor(recommendationHexColor)
.rank(rank)
.description(description)
.isHint(isHint)
@ -67,7 +68,7 @@ public class DictionaryPersistenceService {
typeRepository.findById(typeId).ifPresent((type) -> {
type.setVersion(type.getVersion() + 1);
checkRankAlreadyExists(type.getType(), type.getDossierTemplate().getId(), typeValueRequest.getRank(), type.getDossier() == null ? null : type.getDossier().getId());
BeanUtils.copyProperties(typeValueRequest, type, "dossierTemplateId", "dossierId", "entries", "dossierTemplate", "dossier", "id", "version");
BeanUtils.copyProperties(typeValueRequest, type, "dossierTemplateId", "dossierId", "entries","falsePositiveEntries", "falseRecommendationEntries", "dossierTemplate", "dossier", "id", "version");
});
}

116
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java
View File

@ -1,59 +1,121 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.List;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class EntryPersistenceService {
private final EntryRepository entryRepository;
private final TypeRepository typeRepository;
private final FalsePositiveEntryRepository falsePositiveEntryRepository;
private final FalseRecommendationEntryRepository falseRecommendationEntryRepository;
public void addEntry(String typeId, List<String> entries, long version) {
public void addEntry(String typeId, List<String> entries, long version, DictionaryEntryType dictionaryEntryType) {
var type = typeRepository.getById(typeId);
var dictionaryEntries = entries.stream().map(word -> {
DictionaryEntryEntity entry = new DictionaryEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
entryRepository.saveAll(dictionaryEntries);
switch (dictionaryEntryType) {
case ENTRY:
var dictionaryEntries = entries.stream().map(word -> {
DictionaryEntryEntity entry = new DictionaryEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
entryRepository.saveAll(dictionaryEntries);
break;
case FALSE_POSITIVE:
var dictionaryFalsePositiveEntries = entries.stream().map(word -> {
DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
falsePositiveEntryRepository.saveAll(dictionaryFalsePositiveEntries);
break;
case FALSE_RECOMMENDATION:
var dictionaryFalseRecommendationsEntries = entries.stream().map(word -> {
DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
falseRecommendationEntryRepository.saveAll(dictionaryFalseRecommendationsEntries);
break;
}
}
@Transactional
public void deleteEntries(String typeId, List<String> values, long version) {
entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
public void deleteEntries(String typeId, List<String> values, long version,
DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
}
}
@Transactional
public void setVersion(String typeId, List<String> values, long version) {
entryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
public void setVersion(String typeId, List<String> values, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
break;
}
}
public List<DictionaryEntryEntity> getEntries(String typeId) {
return entryRepository.findByTypeId(typeId);
public List<? extends BaseDictionaryEntry> getEntries(String typeId, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
return entryRepository.findByTypeId(typeId);
case FALSE_POSITIVE:
return falsePositiveEntryRepository.findByTypeId(typeId);
case FALSE_RECOMMENDATION:
return falseRecommendationEntryRepository.findByTypeId(typeId);
}
return null;
}
}

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -39,6 +39,7 @@ public class AddRedactionPersistenceService {
manualRedactionEntry.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
manualRedactionEntry.setPositions(convert(addRedactionRequest.getPositions()));
manualRedactionEntry.setTypeId(addRedactionRequest.getTypeId());
manualRedactionEntry.setDictionaryEntryType(addRedactionRequest.getDictionaryEntryType());
if (addRedactionRequest.getStatus() == AnnotationStatus.APPROVED) {
manualRedactionEntry.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));

30
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java Normal file
View File

@ -0,0 +1,30 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values")
void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List<String> values);
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId")
void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
List<DictionaryFalsePositiveEntryEntity> findByTypeId(String typeId);
}

31
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java Normal file
View File

@ -0,0 +1,31 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
public interface FalseRecommendationEntryRepository extends JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values")
void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List<String> values);
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId")
void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
List<DictionaryFalsePositiveEntryEntity> findByTypeId(String typeId);
}

65
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.controller;
import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities;
import com.iqser.red.service.peristence.v1.server.service.StopwordService;
import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
@ -13,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
@ -56,7 +58,8 @@ public class DictionaryController implements DictionaryResource {
@Override
public void addEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries,
@RequestParam(value = "removeCurrent", required = false, defaultValue = "false") boolean removeCurrent,
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries) {
@RequestParam(value = "ignoreInvalidEntries", required = false, defaultValue = "false") boolean ignoreInvalidEntries,
@RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) {
Set<String> cleanEntries = entries.stream().map(this::cleanDictionaryEntry).collect(toSet());
@ -77,10 +80,10 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
List<String> existing = entryPersistenceService.getEntries(typeId)
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType)
.stream()
.filter(e -> !e.isDeleted())
.map(DictionaryEntryEntity::getValue)
.map(BaseDictionaryEntry::getValue)
.collect(toList());
if (removeCurrent) {
@ -91,8 +94,8 @@ public class DictionaryController implements DictionaryResource {
List<String> added = new ArrayList<>(cleanEntries);
added.removeAll(existing);
entryPersistenceService.deleteEntries(typeId, removed, currentVersion + 1);
entryPersistenceService.addEntry(typeId, added, currentVersion + 1);
entryPersistenceService.deleteEntries(typeId, removed, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.addEntry(typeId, added, currentVersion + 1, dictionaryEntryType);
} else {
@ -101,7 +104,7 @@ public class DictionaryController implements DictionaryResource {
if (added.isEmpty()) {
return;
}
entryPersistenceService.addEntry(typeId, added, currentVersion + 1);
entryPersistenceService.addEntry(typeId, added, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
@ -109,7 +112,7 @@ public class DictionaryController implements DictionaryResource {
@Override
public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries) {
public void deleteEntries(@PathVariable(TYPE_PARAMETER_NAME) String typeId, @RequestBody List<String> entries, @RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) {
// To check whether the type exists
Type typeResult = convert(dictionaryPersistenceService.getType(typeId), Type.class);
@ -117,15 +120,15 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
if (typeResult.isCaseInsensitive()) {
List<String> existing = entryPersistenceService.getEntries(typeId)
List<String> existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY)
.stream()
.map(DictionaryEntryEntity::getValue)
.map(BaseDictionaryEntry::getValue)
.collect(toList());
entryPersistenceService.deleteEntries(typeId, existing.stream()
.filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase))
.collect(toList()), currentVersion + 1);
.collect(toList()), currentVersion + 1, dictionaryEntryType);
} else {
entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1);
entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
@ -155,11 +158,24 @@ public class DictionaryController implements DictionaryResource {
.getRank() != typeValueRequest.getRank()) {
var currentVersion = getCurrentVersion(typeResult);
List<DictionaryEntry> entries = convert(entryPersistenceService.getEntries(typeId), DictionaryEntry.class);
var entries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY), DictionaryEntry.class);
entryPersistenceService.setVersion(typeId, entries.stream()
.filter(entry -> !entry.isDeleted())
.map(DictionaryEntry::getValue)
.collect(toList()), currentVersion + 1);
.collect(toList()), currentVersion + 1, DictionaryEntryType.ENTRY);
var falsePositiveEntries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.FALSE_POSITIVE), DictionaryEntry.class);
entryPersistenceService.setVersion(typeId, falsePositiveEntries.stream()
.filter(entry -> !entry.isDeleted())
.map(DictionaryEntry::getValue)
.collect(toList()), currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
var falseRecommendationEntries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.FALSE_RECOMMENDATION), DictionaryEntry.class);
entryPersistenceService.setVersion(typeId, falseRecommendationEntries.stream()
.filter(entry -> !entry.isDeleted())
.map(DictionaryEntry::getValue)
.collect(toList()), currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
}
}
@ -193,7 +209,7 @@ public class DictionaryController implements DictionaryResource {
}
String color = typeRequest.getHexColor();
validateColor(color);
return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color, typeRequest
return convert(dictionaryPersistenceService.addType(typeRequest.getType(), typeRequest.getDossierTemplateId(), color,typeRequest.getRecommendationHexColor(), typeRequest
.getRank(), typeRequest.isHint(), typeRequest.isCaseInsensitive(), typeRequest.isRecommendation(), typeRequest
.getDescription(), typeRequest.isAddToDictionaryAction(), typeRequest.getLabel(), typeRequest.getDossierId()), Type.class);
}
@ -208,12 +224,12 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
dictionaryPersistenceService.deleteType(typeId);
List<String> existing = entryPersistenceService.getEntries(typeId)
List<String> existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY)
.stream()
.map(DictionaryEntryEntity::getValue)
.map(BaseDictionaryEntry::getValue)
.collect(toList());
entryPersistenceService.deleteEntries(typeId, existing, currentVersion + 1);
entryPersistenceService.deleteEntries(typeId, existing, currentVersion + 1, DictionaryEntryType.ENTRY);
dictionaryPersistenceService.incrementVersion(typeId);
}
@ -239,15 +255,26 @@ public class DictionaryController implements DictionaryResource {
var entity = dictionaryPersistenceService.getType(typeId);
var target = convert(entity, Type.class);
target.setEntries(convert(entity.getEntries(), DictionaryEntry.class));
target.setFalsePositiveEntries(convert(entity.getEntries(), DictionaryEntry.class));
target.setFalseRecommendationEntries(convert(entity.getEntries(), DictionaryEntry.class));
return target;
}
@Override
@Transactional
public List<DictionaryEntry> getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId) {
public List<DictionaryEntry> getEntriesForType(@PathVariable(TYPE_PARAMETER_NAME) String typeId,
@RequestParam(value = "dictionaryEntryType", required = false, defaultValue = "ENTRY") DictionaryEntryType dictionaryEntryType) {
return convert(dictionaryPersistenceService.getType(typeId).getEntries(), DictionaryEntry.class);
switch (dictionaryEntryType) {
case ENTRY:
return convert(dictionaryPersistenceService.getType(typeId).getEntries(), DictionaryEntry.class);
case FALSE_POSITIVE:
return convert(dictionaryPersistenceService.getType(typeId).getFalsePositiveEntries(), DictionaryEntry.class);
case FALSE_RECOMMENDATION:
return convert(dictionaryPersistenceService.getType(typeId).getFalseRecommendationEntries(), DictionaryEntry.class);
}
return null;
}

218
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -60,6 +60,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Resize
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.MessageType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
@ -119,12 +120,12 @@ public class ManualRedactionService {
commentId = addComment(fileId, annotationId, addRedactionRequest.getComment(), addRedactionRequest.getUser()).getId();
}
var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId);
var actionPerformed = handleAddToDictionary(fileId, annotationId, addRedactionRequest.getTypeId(), addRedactionRequest.getValue(), addRedactionRequest.getStatus(), addRedactionRequest.isAddToDictionary(), addRedactionRequest.isAddToDossierDictionary(), false, dossierId, addRedactionRequest.getDictionaryEntryType());
if(actionPerformed){
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId,fileId);
}else{
reprocess(dossierId, fileId);
} else {
if (!addRedactionRequest.isAddToDictionary() && !addRedactionRequest.isAddToDossierDictionary() && !addRedactionRequest.isRectangle()) {
var loaded = convert(getAddRedaction(fileId, annotationId), ManualRedactionEntry.class, new ManualRedactionMapper());
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(Set.of(loaded)).build();
@ -138,14 +139,15 @@ public class ManualRedactionService {
}
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId, List<RemoveRedactionRequest> removeRedactionRequests) {
public List<ManualAddResponse> addRemoveRedaction(String dossierId, String fileId,
List<RemoveRedactionRequest> removeRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
for(var removeRedactionRequest : removeRedactionRequests ){
for (var removeRedactionRequest : removeRedactionRequests) {
removeRedactionPersistenceService.insert(fileId, removeRedactionRequest);
@ -156,35 +158,10 @@ public class ManualRedactionService {
actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog, dossier, fileId, removeRedactionRequest.getAnnotationId(), removeRedactionRequest.getStatus(), removeRedactionRequest.isRemoveFromDictionary(), false);
response.add(ManualAddResponse.builder().annotationId(removeRedactionRequest.getAnnotationId()).commentId(commentId).build());
}
if(actionPerformed){
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId, List<ForceRedactionRequest> forceRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
for(var forceRedactionRequest : forceRedactionRequests ) {
forceRedactionPersistenceService.insert(fileId, forceRedactionRequest);
Long commentId = null;
if (forceRedactionRequest.getComment() != null) {
commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId();
}
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus());
response.add(ManualAddResponse.builder().annotationId(forceRedactionRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build());
}
if (actionPerformed) {
@ -197,12 +174,45 @@ public class ManualRedactionService {
}
public List<ManualAddResponse> addLegalBasisChange(String dossierId, String fileId, List<LegalBasisChangeRequest> legalBasisChangeRequests) {
public List<ManualAddResponse> addForceRedaction(String dossierId, String fileId,
List<ForceRedactionRequest> forceRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
for (var forceRedactionRequest : forceRedactionRequests) {
forceRedactionPersistenceService.insert(fileId, forceRedactionRequest);
Long commentId = null;
if (forceRedactionRequest.getComment() != null) {
commentId = addComment(fileId, forceRedactionRequest.getAnnotationId(), forceRedactionRequest.getComment(), forceRedactionRequest.getUser()).getId();
}
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(forceRedactionRequest.getStatus());
response.add(ManualAddResponse.builder()
.annotationId(forceRedactionRequest.getAnnotationId())
.commentId(commentId)
.build());
}
if (actionPerformed) {
reprocess(dossierId, fileId);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}
public List<ManualAddResponse> addLegalBasisChange(String dossierId, String fileId,
List<LegalBasisChangeRequest> legalBasisChangeRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
for(var legalBasisChangeRequest : legalBasisChangeRequests) {
for (var legalBasisChangeRequest : legalBasisChangeRequests) {
legalBasisChangePersistenceService.insert(fileId, legalBasisChangeRequest);
Long commentId = null;
@ -211,7 +221,10 @@ public class ManualRedactionService {
commentId = addComment(fileId, legalBasisChangeRequest.getAnnotationId(), legalBasisChangeRequest.getComment(), legalBasisChangeRequest.getUser()).getId();
}
response.add(ManualAddResponse.builder().annotationId(legalBasisChangeRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder()
.annotationId(legalBasisChangeRequest.getAnnotationId())
.commentId(commentId)
.build());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
@ -219,13 +232,14 @@ public class ManualRedactionService {
}
public List<ManualAddResponse> addImageRecategorization(String dossierId, String fileId, List<ImageRecategorizationRequest> imageRecategorizationRequests) {
public List<ManualAddResponse> addImageRecategorization(String dossierId, String fileId,
List<ImageRecategorizationRequest> imageRecategorizationRequests) {
var response = new ArrayList<ManualAddResponse>();
var actionPerformed = false;
dossierPersistenceService.getAndValidateDossier(dossierId);
for(var imageRecategorizationRequest : imageRecategorizationRequests) {
for (var imageRecategorizationRequest : imageRecategorizationRequests) {
recategorizationPersistenceService.insert(fileId, imageRecategorizationRequest);
Long commentId = null;
@ -236,7 +250,10 @@ public class ManualRedactionService {
actionPerformed = actionPerformed || !AnnotationStatus.REQUESTED.equals(imageRecategorizationRequest.getStatus());
response.add(ManualAddResponse.builder().annotationId(imageRecategorizationRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder()
.annotationId(imageRecategorizationRequest.getAnnotationId())
.commentId(commentId)
.build());
}
if (actionPerformed) {
reprocess(dossierId, fileId);
@ -306,9 +323,9 @@ public class ManualRedactionService {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var actionPerformed = false;
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
var addRedaction = getAddRedaction(fileId, annotationId);
actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedaction.getTypeId(), addRedaction.getValue(), addRedaction.getStatus(), addRedaction.isAddToDictionary(), addRedaction.isAddToDossierDictionary(), true, dossier.getId());
actionPerformed = actionPerformed || handleAddToDictionary(fileId, annotationId, addRedaction.getTypeId(), addRedaction.getValue(), addRedaction.getStatus(), addRedaction.isAddToDictionary(), addRedaction.isAddToDossierDictionary(), true, dossier.getId(), addRedaction.getDictionaryEntryType());
addRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
@ -326,15 +343,15 @@ public class ManualRedactionService {
var actionPerformed = false;
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), fileId);
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
var removeRedaction = getRemoveRedaction(fileId, annotationId);
actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog , dossier, fileId, annotationId, removeRedaction.getStatus(), removeRedaction.isRemoveFromDictionary(), true);
actionPerformed = actionPerformed || handleRemoveFromDictionary(redactionLog, dossier, fileId, annotationId, removeRedaction.getStatus(), removeRedaction.isRemoveFromDictionary(), true);
removeRedactionPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
if(actionPerformed) {
if (actionPerformed) {
reprocess(dossierId, fileId);
}
@ -346,7 +363,7 @@ public class ManualRedactionService {
var now = OffsetDateTime.now();
for(var annotationId : annotationIds ) {
for (var annotationId : annotationIds) {
forceRedactionPersistenceService.softDelete(fileId, annotationId, now);
}
@ -356,7 +373,7 @@ public class ManualRedactionService {
public void deleteLegalBasisChange(String dossierId, String fileId, List<String> annotationIds) {
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
legalBasisChangePersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
}
@ -367,7 +384,7 @@ public class ManualRedactionService {
public void deleteImageRecategorization(String dossierId, String fileId, List<String> annotationIds) {
var actionPerformed = false;
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
var imageRecategorization = getImageRecategorization(fileId, annotationId);
recategorizationPersistenceService.softDelete(fileId, annotationId, OffsetDateTime.now());
@ -383,8 +400,9 @@ public class ManualRedactionService {
public void deleteComment(String fileId, List<Long> commentIds) {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
for(var commentId : commentIds) {
for (var commentId : commentIds) {
commentPersistenceService.softDelete(commentId, OffsetDateTime.now());
}
// update indicator
@ -395,19 +413,20 @@ public class ManualRedactionService {
public void deleteResizeRedaction(String dossierId, String fileId, List<String> annotationIds) {
OffsetDateTime now = OffsetDateTime.now();
for(var annotationId : annotationIds ) {
for (var annotationId : annotationIds) {
resizeRedactionPersistenceService.softDelete(fileId, annotationId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId, List<ResizeRedactionRequest> resizeRedactionRequests) {
public List<ManualAddResponse> addResizeRedaction(String dossierId, String fileId,
List<ResizeRedactionRequest> resizeRedactionRequests) {
var response = new ArrayList<ManualAddResponse>();
dossierPersistenceService.getAndValidateDossier(dossierId);
for(var resizeRedactionRequest : resizeRedactionRequests ) {
for (var resizeRedactionRequest : resizeRedactionRequests) {
resizeRedactionPersistenceService.insert(fileId, resizeRedactionRequest);
Long commentId = null;
@ -419,7 +438,10 @@ public class ManualRedactionService {
ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of(loaded)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
response.add(ManualAddResponse.builder().annotationId(resizeRedactionRequest.getAnnotationId()).commentId(commentId).build());
response.add(ManualAddResponse.builder()
.annotationId(resizeRedactionRequest.getAnnotationId())
.commentId(commentId)
.build());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
@ -429,16 +451,16 @@ public class ManualRedactionService {
@SuppressWarnings("PMD")
public void updateRemoveRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
public void updateRemoveRedactionStatus(String dossierId, String fileId, List<String> annotationIds,
AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
RedactionLog redactionLog = fileManagementStorageService.getRedactionLog(dossierId, fileId);
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
IdRemovalEntity idRemoval = removeRedactionPersistenceService.findRemoveRedaction(fileId, annotationId);
if (idRemoval.isRemoveFromDictionary()) {
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(idRemoval.getId().getAnnotationId()))
@ -451,14 +473,14 @@ public class ManualRedactionService {
var redactionLogEntry = redactionLogEntryOptional.get();
if (annotationStatus == AnnotationStatus.APPROVED) {
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId);
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
approveStatusForRedactionsWithSameValue(dossier, false, true, redactionLogEntry.getValue());
} else if (annotationStatus == AnnotationStatus.DECLINED) {
// if it was previously approved, revert the delete
if (idRemoval.getStatus() == AnnotationStatus.APPROVED) {
addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId);
addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossierId, fileId, DictionaryEntryType.ENTRY);
}
}
}
@ -472,10 +494,11 @@ public class ManualRedactionService {
}
public void updateForceRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
public void updateForceRedactionStatus(String dossierId, String fileId, List<String> annotationIds,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
@ -491,10 +514,11 @@ public class ManualRedactionService {
@Transactional
public void updateLegalBasisChangeStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
public void updateLegalBasisChangeStatus(String dossierId, String fileId, List<String> annotationIds,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus);
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
@ -503,16 +527,16 @@ public class ManualRedactionService {
}
public void updateImageRecategorizationStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
public void updateImageRecategorizationStatus(String dossierId, String fileId, List<String> annotationIds,
AnnotationStatus annotationStatus) {
var actionPerformed = false;
dossierPersistenceService.getAndValidateDossier(dossierId);
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
ManualImageRecategorizationEntity imageRecategorization = recategorizationPersistenceService.findRecategorization(fileId, annotationId);
actionPerformed = actionPerformed ||
AnnotationStatus.DECLINED.equals(annotationStatus) && AnnotationStatus.APPROVED.equals(imageRecategorization.getStatus());
actionPerformed = actionPerformed || AnnotationStatus.DECLINED.equals(annotationStatus) && AnnotationStatus.APPROVED.equals(imageRecategorization.getStatus());
actionPerformed = actionPerformed || AnnotationStatus.APPROVED.equals(annotationStatus);
recategorizationPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
@ -537,7 +561,8 @@ public class ManualRedactionService {
}
public void updateResizeRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
public void updateResizeRedactionStatus(String dossierId, String fileId, List<String> annotationIds,
AnnotationStatus annotationStatus) {
dossierPersistenceService.getAndValidateDossier(dossierId);
for (var annotationId : annotationIds) {
@ -550,26 +575,28 @@ public class ManualRedactionService {
@SuppressWarnings("PMD")
public void updateAddRedactionStatus(String dossierId, String fileId, List<String> annotationIds, AnnotationStatus annotationStatus) {
public void updateAddRedactionStatus(String dossierId, String fileId, List<String> annotationIds,
AnnotationStatus annotationStatus) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
for(var annotationId : annotationIds) {
for (var annotationId : annotationIds) {
ManualRedactionEntryEntity manualRedactionEntry = addRedactionPersistenceService.findAddRedaction(fileId, annotationId);
if (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) {
if (annotationStatus == AnnotationStatus.APPROVED) {
addToDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId);
addToDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId, manualRedactionEntry.getDictionaryEntryType());
approveStatusForRedactionsWithSameValue(dossier, manualRedactionEntry.isAddToDictionary(), manualRedactionEntry.isAddToDossierDictionary(), manualRedactionEntry.getValue());
} else if (annotationStatus == AnnotationStatus.DECLINED) {
// if it was previously approved, revert the add
if (manualRedactionEntry.getStatus() == AnnotationStatus.APPROVED) {
removeFromDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId);
removeFromDictionary(manualRedactionEntry.getTypeId(), manualRedactionEntry.getValue(), dossierId, fileId, manualRedactionEntry.getDictionaryEntryType());
}
}
}
addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
addRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS));
// boolean hasSuggestions = calculateHasSuggestions(fileId);
// fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasSuggestions);
@ -591,7 +618,8 @@ public class ManualRedactionService {
manualRedactions.getForceRedactions().forEach(forceRedactionPersistenceService::markAsProcessed);
}
if (manualRedactions.getImageRecategorization() != null) {
manualRedactions.getImageRecategorization().forEach(recategorizationPersistenceService::markAsProcessed);
manualRedactions.getImageRecategorization()
.forEach(recategorizationPersistenceService::markAsProcessed);
}
}
}
@ -601,7 +629,12 @@ public class ManualRedactionService {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.SURROUNDING_TEXT_PROCESSING);
var analyseRequest = AnalyzeRequest.builder().messageType(MessageType.SURROUNDING_TEXT).dossierId(dossierId).fileId(fileId).manualRedactions(manualRedactions).build();
var analyseRequest = AnalyzeRequest.builder()
.messageType(MessageType.SURROUNDING_TEXT)
.dossierId(dossierId)
.fileId(fileId)
.manualRedactions(manualRedactions)
.build();
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest), message -> {
@ -615,11 +648,13 @@ public class ManualRedactionService {
}
private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary, boolean addToDossierDictionary, String value) {
private void approveStatusForRedactionsWithSameValue(DossierEntity dossier, boolean addToDictionary,
boolean addToDossierDictionary, String value) {
List<DossierEntity> dossiers = new ArrayList<>();
if (addToDictionary) {
dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId()).getDossiers();
dossiers = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId())
.getDossiers();
}
if (addToDossierDictionary) {
@ -645,13 +680,14 @@ public class ManualRedactionService {
}
private boolean handleRemoveFromDictionary(RedactionLog redactionLog, DossierEntity dossier, String fileId, String annotationId, AnnotationStatus status, boolean removeFromDictionary, boolean revert) {
private boolean handleRemoveFromDictionary(RedactionLog redactionLog, DossierEntity dossier, String fileId,
String annotationId, AnnotationStatus status,
boolean removeFromDictionary, boolean revert) {
if (status == AnnotationStatus.APPROVED) {
if (removeFromDictionary) {
Optional<RedactionLogEntry> redactionLogEntryOptional = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getId().equals(annotationId))
@ -664,9 +700,9 @@ public class ManualRedactionService {
var redactionLogEntry = redactionLogEntryOptional.get();
if (revert) {
addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId);
addToDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY);
} else {
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId);
removeFromDictionary(buildTypeId(redactionLogEntry, dossier), redactionLogEntry.getValue(), dossier.getId(), fileId, DictionaryEntryType.ENTRY);
}
return true;
@ -677,22 +713,24 @@ public class ManualRedactionService {
}
private void addToDictionary(String typeId, String value, String dossierId, String fileId) {
private void addToDictionary(String typeId, String value, String dossierId, String fileId,
DictionaryEntryType dictionaryEntryType) {
try {
log.debug("Adding entry: {} to {} for {} / {}", value, typeId, dossierId, fileId);
dictionaryController.addEntries(typeId, List.of(value), false, false);
dictionaryController.addEntries(typeId, List.of(value), false, false, dictionaryEntryType != null ? dictionaryEntryType : DictionaryEntryType.ENTRY);
} catch (Exception e) {
throw new BadRequestException(e.getMessage());
}
}
private void removeFromDictionary(String typeId, String value, String dossierId, String fileId) {
private void removeFromDictionary(String typeId, String value, String dossierId, String fileId,
DictionaryEntryType dictionaryEntryType) {
try {
log.debug("Deleting entries to {} for {} / {}", typeId, dossierId, fileId);
dictionaryController.deleteEntries(typeId, List.of(value));
dictionaryController.deleteEntries(typeId, List.of(value), dictionaryEntryType != null ? dictionaryEntryType : DictionaryEntryType.ENTRY);
} catch (FeignException e) {
throw new BadRequestException(e.getMessage());
}
@ -711,17 +749,19 @@ public class ManualRedactionService {
}
private boolean handleAddToDictionary(String fileId, String annotationId, String typeId, String value, AnnotationStatus status, boolean addToDictionary,
boolean addToDossierDictionary, boolean revert, String dossierId) {
private boolean handleAddToDictionary(String fileId, String annotationId, String typeId, String value,
AnnotationStatus status, boolean addToDictionary,
boolean addToDossierDictionary, boolean revert, String dossierId,
DictionaryEntryType dictionaryEntryType) {
if (status == AnnotationStatus.APPROVED) {
addRedactionPersistenceService.updateStatus(fileId, annotationId, status, addToDictionary, addToDossierDictionary);
if (addToDictionary || addToDossierDictionary) {
if (revert) {
removeFromDictionary(typeId, value, dossierId, fileId);
removeFromDictionary(typeId, value, dossierId, fileId, dictionaryEntryType);
} else {
addToDictionary(typeId, value, dossierId, fileId);
addToDictionary(typeId, value, dossierId, fileId, dictionaryEntryType);
}
return true;
}

111
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml Normal file
View File

@ -0,0 +1,111 @@
databaseChangeLog:
- changeSet:
id: add-recommendaction-color
author: dom
changes:
- addColumn:
columns:
- column:
name: recommendation_hex_color
type: VARCHAR(255)
tableName: type
- changeSet:
id: 11-added-dictionary_false_positive_table
author: dom
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
primaryKeyName: dictionary_false_positive_entry_pkey
name: entry_id
type: BIGINT
- column:
name: deleted
type: BOOLEAN
- column:
name: value
type: VARCHAR(4000)
- column:
name: version
type: BIGINT
- column:
name: type_id
type: VARCHAR(255)
tableName: dictionary_false_positive_entry
- changeSet:
id: 11-added-dictionary_false_positive_table-type-fk
author: dom
changes:
- addForeignKeyConstraint:
baseColumnNames: type_id
baseTableName: dictionary_false_positive_entry
constraintName: dictionary_false_positive_table-type-fk-constraint
deferrable: false
initiallyDeferred: false
onDelete: NO ACTION
onUpdate: NO ACTION
referencedColumnNames: id
referencedTableName: type
validate: true
- changeSet:
id: 11-dictionary_false_recommendation_entry_table
author: dom
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
primaryKeyName: dictionary_false_recommendation_entry_pkey
name: entry_id
type: BIGINT
- column:
name: deleted
type: BOOLEAN
- column:
name: value
type: VARCHAR(4000)
- column:
name: version
type: BIGINT
- column:
name: type_id
type: VARCHAR(255)
tableName: dictionary_false_recommendation_entry
- changeSet:
id: 11-dictionary_false_recommendation_entry_table-type-fk
author: dom
changes:
- addForeignKeyConstraint:
baseColumnNames: type_id
baseTableName: dictionary_false_recommendation_entry
constraintName: dictionary_false_recommendation_entry_table-type-fk-constraint
deferrable: false
initiallyDeferred: false
onDelete: NO ACTION
onUpdate: NO ACTION
referencedColumnNames: id
referencedTableName: type
validate: true
- changeSet:
id: rename-type-table-to-entity
author: dom
changes:
- renameTable:
oldTableName: type
newTableName: entity
- changeSet:
id: added-dictionaryEntryType-to-manual-redaction
author: dom
changes:
- addColumn:
columns:
- column:
name: dictionary_entry_type
type: VARCHAR(255)
defaultValue: ENTRY
tableName: manual_redaction

4
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -22,4 +22,6 @@ databaseChangeLog:
- include:
file: db/changelog/10-added-file-manipulation-date.changelog.yaml
- include:
file: db/changelog/sql/10-set-file-manipulation-date.sql
file: db/changelog/sql/10-set-file-manipulation-date.sql
- include:
file: db/changelog/11-added-dictionary_false_positive_tables.changelog.yaml

1
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/TypeProvider.java
View File

@ -23,6 +23,7 @@ public class TypeProvider {
type.setDescription("test");
type.setAddToDictionaryAction(true);
type.setHexColor("#dddddd");
type.setRecommendationHexColor("#aaaaaa");
type.setHint(false);
type.setRank(100);
type.setRecommendation(false);

95
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.Test;
@ -14,6 +15,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import feign.FeignException;
@ -36,6 +38,27 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
private DossierTemplateClient dossierTemplateClient;
@Test
public void testAddFalsePositiveAndFalseRecommendation() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var type = typeProvider.testAndProvideType(dossierTemplate);
assertThat(type.getRecommendationHexColor()).isEqualTo("#aaaaaa");
dictionaryClient.addEntries(type.getTypeId(), List.of("word1", "word2"), false, false, DictionaryEntryType.ENTRY);
dictionaryClient.addEntries(type.getTypeId(), List.of("false_positive1", "false_positive"), false, false, DictionaryEntryType.FALSE_POSITIVE);
dictionaryClient.addEntries(type.getTypeId(), List.of("false_recommendation1", "false_recommendation2"), false, false, DictionaryEntryType.FALSE_RECOMMENDATION);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(2);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.FALSE_POSITIVE)).hasSize(2);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.FALSE_RECOMMENDATION)).hasSize(2);
var loadedType = dictionaryClient.getDictionaryForType(type.getTypeId());
assertThat(loadedType.getEntries()).hasSize(2);
assertThat(loadedType.getFalsePositiveEntries()).hasSize(2);
assertThat(loadedType.getFalseRecommendationEntries()).hasSize(2);
}
@Test
public void testAddEntriesWithStopWord() {
@ -45,7 +68,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add("age");
entries.add("page");
try {
dictionaryClient.addEntries(typeId, entries, false, false);
dictionaryClient.addEntries(typeId, entries, false, false, DictionaryEntryType.ENTRY);
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(400);
}
@ -64,18 +87,18 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
var entries = new ArrayList<String>();
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
// Act & Assert: Add same word again; Only one should exist
entries = new ArrayList<>();
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
// Act & Assert: Add same word multiple times again; Only one should exist
entries = new ArrayList<>();
@ -83,19 +106,19 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word);
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
// Act & Assert: Delete word; Should have 'deleted' flag
entries = new ArrayList<>();
entries.add(word);
dictionaryClient.deleteEntries(type.getTypeId(), entries);
dictionaryClient.deleteEntries(type.getTypeId(), entries, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).isDeleted()).isTrue();
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).isDeleted()).isTrue();
}
@ -111,18 +134,18 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
var entries = new ArrayList<String>();
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
// Act & Assert: Add same word again; Only one should exist
entries = new ArrayList<>();
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
// Act & Assert: Add same word multiple times again; Only one should exist
entries = new ArrayList<>();
@ -130,19 +153,19 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word);
entries.add(word);
entries.add(word);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false);
dictionaryClient.addEntries(type.getTypeId(), entries, true, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
// Act & Assert: Delete word; Should have 'deleted' flag
entries = new ArrayList<>();
entries.add(word);
dictionaryClient.deleteEntries(type.getTypeId(), entries);
dictionaryClient.deleteEntries(type.getTypeId(), entries, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId()).get(0).isDeleted()).isTrue();
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(1);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).getValue()).isEqualTo(word);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY).get(0).isDeleted()).isTrue();
}
@ -161,10 +184,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word1);
entries.add(word2);
entries.add(word3);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)
.stream()
.map(e -> e.getValue())
.collect(Collectors.toList())).contains(word1, word2, word3);
@ -172,10 +195,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
// Act & Assert: Add same word again; No duplicate should exist
entries = new ArrayList<>();
entries.add(word1);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)
.stream()
.map(e -> e.getValue())
.collect(Collectors.toList())).contains(word1, word2, word3);
@ -188,10 +211,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
entries.add(word1);
entries.add(word2);
entries.add(word3);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false);
dictionaryClient.addEntries(type.getTypeId(), entries, false, false, DictionaryEntryType.ENTRY);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId())
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)).hasSize(3);
assertThat(dictionaryClient.getEntriesForType(type.getTypeId(), DictionaryEntryType.ENTRY)
.stream()
.map(e -> e.getValue())
.collect(Collectors.toList())).contains(word1, word2, word3);

15
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java
View File

@ -11,6 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.Do
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStats;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import org.junit.Test;
@ -77,9 +78,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var entries1 = new ArrayList<String>();
entries1.add("entry1");
entries1.add("entry2");
dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false);
dictionaryClient.addEntries(addedType.getTypeId(), entries1, false, false, DictionaryEntryType.ENTRY);
List<DictionaryEntry> entryList = dictionaryClient.getEntriesForType(addedType.getTypeId());
List<DictionaryEntry> entryList = dictionaryClient.getEntriesForType(addedType.getTypeId(), DictionaryEntryType.ENTRY);
assertThat(entryList.size()).isEqualTo(entries1.size());
var addedType2 = dictionaryClient.addType(Type.builder()
@ -96,9 +97,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
entries2.add("entry1");
entries2.add("entry2");
entries2.add("entry3");
dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false);
dictionaryClient.addEntries(addedType2.getTypeId(), entries2, false, false, DictionaryEntryType.ENTRY);
entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId());
entryList = dictionaryClient.getEntriesForType(addedType2.getTypeId(), DictionaryEntryType.ENTRY);
assertThat(entryList.size()).isEqualTo(entries2.size());
var dossierTemplate3 = provideTestTemplate("dossierTemp3");
@ -117,9 +118,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var entries3 = new ArrayList<String>();
entries3.add("entry1");
entries3.add("entry2");
dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false);
dictionaryClient.addEntries(addedType3.getTypeId(), entries3, false, false, DictionaryEntryType.ENTRY);
entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId());
entryList = dictionaryClient.getEntriesForType(addedType3.getTypeId(), DictionaryEntryType.ENTRY);
assertThat(entryList.size()).isEqualTo(entries3.size());
var dossierTemplate4 = provideTestTemplate("dossierTemp4");
@ -149,7 +150,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var entries22 = new ArrayList<String>();
entries22.add(entries2.get(2));
dictionaryClient.deleteEntries(addedType2.getTypeId(), entries22);
dictionaryClient.deleteEntries(addedType2.getTypeId(), entries22, DictionaryEntryType.ENTRY);
dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateStats(dossierTemplateIds);
assertThat(dossierTemplateStatsList.size()).isEqualTo(dossierTemplateIds.size());

9
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/TypeTest.java
View File

@ -6,6 +6,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import org.assertj.core.util.Lists;
import org.junit.Test;
@ -35,7 +36,7 @@ public class TypeTest extends AbstractPersistenceServerServiceTest {
var type = typeProvider.testAndProvideType(dossierTemplate);
dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false);
dictionaryClient.addEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), true, false, DictionaryEntryType.ENTRY);
var loadedType = dictionaryClient.getDictionaryForType(type.getId());
@ -43,17 +44,17 @@ public class TypeTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedType.getVersion()).isGreaterThan(type.getVersion());
var dict = dictionaryClient.getEntriesForType(type.getId());
var dict = dictionaryClient.getEntriesForType(type.getId(), DictionaryEntryType.ENTRY);
assertThat(dict.size()).isEqualTo(3);
dictionaryClient.deleteEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"));
dictionaryClient.deleteEntries(type.getId(), Lists.newArrayList("aaa", "bbb", "ccc"), DictionaryEntryType.ENTRY);
loadedType = dictionaryClient.getDictionaryForType(type.getId());
assertThat(loadedType.getVersion()).isGreaterThan(type.getVersion() + 1);
dict = dictionaryClient.getEntriesForType(type.getId());
dict = dictionaryClient.getEntriesForType(type.getId(), DictionaryEntryType.ENTRY);
assertThat(dict.size()).isEqualTo(3);
for (var entry : dict) {