From 18d98ab4e6c410207cdd6d6f702c2eabcb44ae1a Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 27 Oct 2023 11:55:38 +0200 Subject: [PATCH 1/3] RED-7384: create shared models in persistence service --- .../processor/service/DictionaryService.java | 34 ++++++++------- .../analysislog/migration/MigratedIds.java | 43 +++++++++++++++++++ .../dossier/file/FileType.java | 4 +- 3 files changed, 64 insertions(+), 17 deletions(-) create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java index 9b740aa56..781589c19 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java @@ -35,7 +35,6 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.utils.ColorUtils; -import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue; import com.iqser.red.service.persistence.service.v1.api.shared.model.Dictionary; import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeResponse; @@ -44,6 +43,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.UpdateTypeV import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -307,6 +307,7 @@ public class DictionaryService { } } + @PreAuthorize("hasAuthority('" + READ_DICTIONARY_TYPES + "')") public Dictionary getMergedDictionaryForType(String type, String dossierTemplateId, String dossierId) { @@ -327,7 +328,9 @@ public class DictionaryService { } } - private Dictionary convertMergedDictionaries (List commonsDictionaries, String dossierTemplateId, String dossierId) { + + private Dictionary convertMergedDictionaries(List commonsDictionaries, String dossierTemplateId, String dossierId) { + List dictionaries = new ArrayList<>(); commonsDictionaries.forEach(cdm -> { var typeId = toTypeId(cdm.getType(), dossierTemplateId, dossierId); @@ -373,27 +376,29 @@ public class DictionaryService { return dictionaries.get(0); } + private List convertType(TypeEntity typeEntity) { return List.of(CommonsDictionaryModel.builder() - .type(typeEntity.getType()) - .rank(typeEntity.getRank()) - .color(ColorUtils.convertColor(typeEntity.getHexColor())) - .caseInsensitive(typeEntity.isCaseInsensitive()) - .hint(typeEntity.isHint()) - .entries(getCommonsEntries(typeEntity.getId(), DictionaryEntryType.ENTRY)) - .falsePositives(getCommonsEntries(typeEntity.getId(), DictionaryEntryType.FALSE_POSITIVE)) - .falseRecommendations(getCommonsEntries(typeEntity.getId(), DictionaryEntryType.FALSE_RECOMMENDATION)) - .build()); + .type(typeEntity.getType()) + .rank(typeEntity.getRank()) + .color(ColorUtils.convertColor(typeEntity.getHexColor())) + .caseInsensitive(typeEntity.isCaseInsensitive()) + .hint(typeEntity.isHint()) + .entries(getCommonsEntries(typeEntity.getId(), DictionaryEntryType.ENTRY)) + .falsePositives(getCommonsEntries(typeEntity.getId(), DictionaryEntryType.FALSE_POSITIVE)) + .falseRecommendations(getCommonsEntries(typeEntity.getId(), DictionaryEntryType.FALSE_RECOMMENDATION)) + .build()); } + private Set getCommonsEntries(String typeId, DictionaryEntryType entryType) { + var entries = MagicConverter.convert(entryPersistenceService.getEntries(typeId, entryType, null), DictionaryEntry.class); - return CollectionUtils.isNotEmpty(entries) ? new HashSet<>(entries.stream() - .map(DictionaryEntryModel::new) - .collect(Collectors.toSet())) : new HashSet<>(); + return CollectionUtils.isNotEmpty(entries) ? new HashSet<>(entries.stream().map(DictionaryEntryModel::new).collect(Collectors.toSet())) : new HashSet<>(); } + @PreAuthorize("hasAuthority('" + WRITE_COLORS + "')") public void setColors(String dossierTemplateId, Colors colors) { //validate all colors before setting them @@ -422,5 +427,4 @@ public class DictionaryService { return MagicConverter.convert(colorsService.getColors(dossierTemplateId), Colors.class); } - } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java new file mode 100644 index 000000000..09752f243 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/migration/MigratedIds.java @@ -0,0 +1,43 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class MigratedIds { + + List mappings; + + + public Map buildOldToNewMapping() { + + return mappings.stream().collect(Collectors.toMap(Mapping::oldId, Mapping::newId)); + } + + + public Map buildNewToOldMapping() { + + return mappings.stream().collect(Collectors.toMap(Mapping::newId, Mapping::oldId)); + } + + + public void addMapping(String oldId, String newId) { + + mappings.add(new Mapping(oldId, newId)); + } + + + private record Mapping(String oldId, String newId) { + + } + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index 8008340ee..f4df3c82c 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -9,7 +9,6 @@ public enum FileType { VIEWER_DOCUMENT(".pdf"), REDACTION_LOG(".json"), SIMPLIFIED_TEXT(".json"), - SECTION_GRID(".json"), TEXT(".json"), // deprecated file type, only present in legacy migrations NER_ENTITIES(".json"), IMAGE_INFO(".json"), @@ -24,7 +23,8 @@ public enum FileType { DOCUMENT_POSITION(".json"), DOCUMENT_PAGES(".json"), ENTITY_LOG(".json"), - COMPONENT_LOG(".json"); + COMPONENT_LOG(".json"), + MIGRATED_IDS(".json"); @Getter private final String extension; -- 2.47.2 From d91bbc6d0e614e79a41a58f60850b321f968e11e Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 27 Oct 2023 12:09:48 +0200 Subject: [PATCH 2/3] RED-7384: create shared models in persistence service --- .../internal/AdminInterfaceController.java | 4 +--- .../service/FileManagementStorageService.java | 14 -------------- .../v1/processor/service/FileStatusService.java | 2 -- .../dossiertemplate/dossier/file/FileType.java | 4 +++- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java index 601653c2e..b4c77e297 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java @@ -22,7 +22,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @RestController @RequiredArgsConstructor -@RequestMapping(InternalApi.BASE_PATH+"/admin") +@RequestMapping(InternalApi.BASE_PATH + "/admin") public class AdminInterfaceController { private final FileManagementStorageService fileManagementStorageService; @@ -35,7 +35,6 @@ public class AdminInterfaceController { @PostMapping("/reset-file") public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); @@ -138,7 +137,6 @@ public class AdminInterfaceController { var dossierId = file.getDossierId(); var fileId = file.getId(); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index 55b17bcbf..96be2eaf2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -18,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -116,19 +115,6 @@ public class FileManagementStorageService { } - public SectionGrid getSectionGrid(String dossierId, String fileId) { - - try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class); - } catch (StorageObjectDoesNotExist e) { - log.debug("SectionGrid not available."); - throw new NotFoundException("SectionGrid does not exist"); - } catch (Exception e) { - throw new RuntimeException("Could not convert SectionGrid", e); - } - } - - public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { try { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index f53861151..474fdfcb1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -640,7 +640,6 @@ public class FileStatusService { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); @@ -670,7 +669,6 @@ public class FileStatusService { // remove everything related to redaction fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); // wipe comments diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java index f4df3c82c..7e67f33ac 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileType.java @@ -24,7 +24,9 @@ public enum FileType { DOCUMENT_PAGES(".json"), ENTITY_LOG(".json"), COMPONENT_LOG(".json"), - MIGRATED_IDS(".json"); + MIGRATED_IDS(".json"), + @Deprecated(forRemoval = true) // still needed for migration to delete existing ones, can be removed as soon as migration is done + SECTION_GRID(".json"); @Getter private final String extension; -- 2.47.2 From a279a7665944b05f34996b07a704effb5174cdf0 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 27 Oct 2023 12:13:47 +0200 Subject: [PATCH 3/3] RED-7384: create shared models in persistence service --- .../service/FileManagementStorageService.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index 96be2eaf2..55b17bcbf 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -18,6 +18,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -115,6 +116,19 @@ public class FileManagementStorageService { } + public SectionGrid getSectionGrid(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class); + } catch (StorageObjectDoesNotExist e) { + log.debug("SectionGrid not available."); + throw new NotFoundException("SectionGrid does not exist"); + } catch (Exception e) { + throw new RuntimeException("Could not convert SectionGrid", e); + } + } + + public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { try { -- 2.47.2