evaluation whether ids stay the same when switching from json to protobuf

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -313,6 +313,10 @@ public class RedactionStorageService {
 
             return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
         }
+        public static String getStorageId(String userId, String dossierId, String filename) {
+
+            return userId + "/" + dossierId + "/" + filename;
+        }
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -11,6 +11,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.nio.file.FileVisitOption;
@@ -41,6 +42,7 @@ import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.data.redis.listener.RedisMessageListenerContainer;
+import org.springframework.data.util.Pair;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -52,6 +54,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
@@ -68,12 +71,24 @@ import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils;
 import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
 import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor;
 import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantProvider;
 import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
@@ -142,12 +157,113 @@ import lombok.extern.slf4j.Slf4j;
     @Autowired
     RedactionStorageService redactionStorageService;
 
+    private static EntryDataProto.EntryData convertEntryData(DocumentStructure.EntryData oldEntryData) {
+        EntryDataProto.EntryData.Builder builder = EntryDataProto.EntryData.newBuilder();
+
+        builder.setType(NodeTypeProto.NodeType.valueOf(oldEntryData.getType().name()));
+        builder.addAllTreeId(Arrays.stream(oldEntryData.getTreeId()).boxed().collect(Collectors.toList()));
+        builder.addAllAtomicBlockIds(Arrays.asList(oldEntryData.getAtomicBlockIds()));
+        builder.addAllPageNumbers(Arrays.asList(oldEntryData.getPageNumbers()));
+
+        builder.putAllProperties(oldEntryData.getProperties());
+
+        // Recursively converting children
+        if (oldEntryData.getChildren() != null) {
+            oldEntryData.getChildren().forEach(child -> builder.addChildren(convertEntryData(child)));
+        }
+
+        return builder.build();
+    }
+
+    private static DocumentStructureProto.DocumentStructure convertDocumentStructure(DocumentStructure oldStructure) {
+        DocumentStructureProto.DocumentStructure.Builder newBuilder = DocumentStructureProto.DocumentStructure.newBuilder();
+
+        if (oldStructure.getRoot() != null) {
+            newBuilder.setRoot(convertEntryData(oldStructure.getRoot()));
+        }
+
+        return newBuilder.build();
+    }
+
+    private static DocumentPageProto.DocumentPage convertDocumentPage(DocumentPage oldPage) {
+        return DocumentPageProto.DocumentPage.newBuilder()
+                .setNumber(oldPage.getNumber())
+                .setHeight(oldPage.getHeight())
+                .setWidth(oldPage.getWidth())
+                .setRotation(oldPage.getRotation())
+                .build();
+    }
+
+    private static DocumentPageProto.AllDocumentPages convertAllDocumentPages(DocumentPage[] oldPages) {
+        DocumentPageProto.AllDocumentPages.Builder allPagesBuilder = DocumentPageProto.AllDocumentPages.newBuilder();
+
+        for (DocumentPage oldPage : oldPages) {
+            DocumentPageProto.DocumentPage newPage = convertDocumentPage(oldPage);
+            allPagesBuilder.addDocumentPages(newPage);
+        }
+
+        return allPagesBuilder.build();
+    }
+
+    private static DocumentPositionDataProto.DocumentPositionData convertPositionData(DocumentPositionData oldData) {
+        DocumentPositionDataProto.DocumentPositionData.Builder builder = DocumentPositionDataProto.DocumentPositionData.newBuilder()
+                .setId(oldData.getId())
+                .addAllStringIdxToPositionIdx(Arrays.stream(oldData.getStringIdxToPositionIdx()).boxed().collect(Collectors.toList()));
+
+        // Convert positions array to Position messages
+        for (float[] pos : oldData.getPositions()) {
+            DocumentPositionDataProto.DocumentPositionData.Position position = DocumentPositionDataProto.DocumentPositionData.Position.newBuilder()
+                    .setX(pos[0])
+                    .setY(pos[1])
+                    .setWidth(pos[2])
+                    .setHeight(pos[3])
+                    .build();
+            builder.addPositions(position);
+        }
+
+        return builder.build();
+    }
+
+    private static DocumentPositionDataProto.AllDocumentPositionData convertAllPositionData(DocumentPositionData[] oldDataList) {
+        DocumentPositionDataProto.AllDocumentPositionData.Builder allDataBuilder = DocumentPositionDataProto.AllDocumentPositionData.newBuilder();
+
+        for (DocumentPositionData oldData : oldDataList) {
+            allDataBuilder.addDocumentPositionData(convertPositionData(oldData));
+        }
+
+        return allDataBuilder.build();
+    }
+
+    public static DocumentTextDataProto.DocumentTextData convertTextData(DocumentTextData oldData) {
+        DocumentTextDataProto.DocumentTextData.Builder builder = DocumentTextDataProto.DocumentTextData.newBuilder()
+                .setId(oldData.getId())
+                .setPage(oldData.getPage())
+                .setSearchText(oldData.getSearchText())
+                .setNumberOnPage(oldData.getNumberOnPage())
+                .setStart(oldData.getStart())
+                .setEnd(oldData.getEnd())
+                .addAllLineBreaks(Arrays.stream(oldData.getLineBreaks()).boxed().collect(Collectors.toList()));
+
+        return builder.build();
+    }
+
+    public static DocumentTextDataProto.AllDocumentTextData convertAllTextData(DocumentTextData[] oldDataList) {
+        DocumentTextDataProto.AllDocumentTextData.Builder allDataBuilder = DocumentTextDataProto.AllDocumentTextData.newBuilder();
+
+        for (DocumentTextData oldData : oldDataList) {
+            allDataBuilder.addDocumentTextData(convertTextData(oldData));
+        }
+
+        return allDataBuilder.build();
+    }
+
+
 
     @Test
     @SneakyThrows
     public void runAnalysisEnd2End() {
 
-        String folder = "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/files_end2end/file_large2"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
+        String folder = "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/files_end2end/file_large"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
 
         Path absoluteFolderPath;
         if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path
@@ -169,15 +285,78 @@ import lombok.extern.slf4j.Slf4j;
                 ObjectMapper objectMapper = new ObjectMapper();
                 objectMapper.registerModule(new JavaTimeModule());
 
-                //analyzeService.analyze(analyzeRequest);
+                String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
-                //storeEntityLogAsTempFile(analyzeRequest, objectMapper);
+                                                                                   analyzeRequest.getFileId(),
+                                                                                   FileType.DOCUMENT_STRUCTURE.name() + ".json");
+                String newDocumentStructureStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_STRUCTURE);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentStructureStorageId)) {
+                    DocumentStructure oldDocumentStructure = objectMapper.readValue(inputStream, DocumentStructure.class);
+                    DocumentStructureProto.DocumentStructure newDocumentStructure = convertDocumentStructure(oldDocumentStructure);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentStructureStorageId, newDocumentStructure);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentStructureStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document structure found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
 
-                restoreEntityLogFromTempFile(objectMapper, analyzeRequest);
+                String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_PAGES.name() + ".json");
+                String newDocumentPagesStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_PAGES);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentPagesStorageId)) {
+                    DocumentPage[] oldDocumentPages = objectMapper.readValue(inputStream, DocumentPage[].class);
+                    DocumentPageProto.AllDocumentPages newDocumentPages = convertAllDocumentPages(oldDocumentPages);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPagesStorageId, newDocumentPages);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPagesStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document pages found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
 
-                addManualRedactionEntryToFile(analyzeRequest);
+                String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
+                                                                                      analyzeRequest.getFileId(),
+                                                                                      FileType.DOCUMENT_POSITION.name() + ".json");
+                String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_POSITION);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentPositionDataStorageId)) {
+                    DocumentPositionData[] oldDocumentPositionData = objectMapper.readValue(inputStream, DocumentPositionData[].class);
+                    DocumentPositionDataProto.AllDocumentPositionData newDocumentPositionData = convertAllPositionData(oldDocumentPositionData);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId, newDocumentPositionData);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document position data found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
+
+                String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
+                                                                                  analyzeRequest.getFileId(),
+                                                                                  FileType.DOCUMENT_TEXT.name() + ".json");
+                String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_TEXT);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentTextDataStorageId)) {
+                    DocumentTextData[] oldDocumentTextData = objectMapper.readValue(inputStream, DocumentTextData[].class);
+                    DocumentTextDataProto.AllDocumentTextData newDocumentTextData = convertAllTextData(oldDocumentTextData);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextData);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document text data found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
 
                 analyzeService.reanalyze(analyzeRequest);
+                storeEntityLogAsTempFile(analyzeRequest, objectMapper);
+
+                EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
                 times.add(System.currentTimeMillis() - start);
+
+                EntityLog otherEntityLog = getEntityLogFromTemp("/tmp/entityLog2.ser", objectMapper);
+
+                List<String> entryIds = entityLog.getEntityLogEntry()
+                        .stream()
+                        .map(EntityLogEntry::getId)
+                        .toList();
+                List<String> otherEntryIds = otherEntityLog.getEntityLogEntry()
+                        .stream()
+                        .map(EntityLogEntry::getId)
+                        .toList();
+                if(entryIds.containsAll(otherEntryIds) && otherEntryIds.containsAll(entryIds)) {
+                    System.out.println("We won");
+                } else {
+                    System.out.println("We lost");
+                }
+
             }
             System.out.println("times in ms for each analyze run: " + times);
         }
@@ -202,7 +381,7 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
-    private void restoreEntityLogFromTempFile(ObjectMapper objectMapper, AnalyzeRequest analyzeRequest) throws JsonProcessingException {
+    private void restoreEntityLogFromTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
         //Deserialize entityLog from a file
         String entityLogString = null;
         try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
@@ -214,6 +393,17 @@ import lombok.extern.slf4j.Slf4j;
         redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
     }
 
+    private EntityLog getEntityLogFromTemp(String fileName, ObjectMapper objectMapper) throws JsonProcessingException {
+        //Deserialize entityLog from a file
+        String entityLogString = null;
+        try (FileInputStream fileIn = new FileInputStream(fileName); ObjectInputStream in = new ObjectInputStream(fileIn)) {
+            entityLogString = (String) in.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        return objectMapper.readValue(entityLogString, EntityLog.class);
+    }
+
 
     private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
 
@@ -222,8 +412,7 @@ import lombok.extern.slf4j.Slf4j;
         // Serialize
         String jsonString = objectMapper.writeValueAsString(entityLog);
         // Serialize entityLog to a file
-        try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser");
+        try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
-             ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
             out.writeObject(jsonString);
         } catch (IOException e) {
             e.printStackTrace();
@@ -274,6 +463,8 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
+
+
     @SneakyThrows
     private List<AnalyzeRequest> prepareStorageForFolder(Path folder) {
 
@@ -307,28 +498,27 @@ import lombok.extern.slf4j.Slf4j;
             request.setManualRedactions(new ManualRedactions());
         }
 
-        Set<FileType> endingsToUpload = Set.of("ORIGIN",
+        Set<FileType> endingsToUpload = Set.of("ORIGIN", "IMAGE_INFO", "NER_ENTITIES", "TABLES", "IMPORTED_REDACTIONS")
-                                               "DOCUMENT_PAGES",
-                                               "DOCUMENT_POSITION",
-                                               "DOCUMENT_STRUCTURE",
-                                               "DOCUMENT_TEXT",
-                                               "IMAGE_INFO",
-                                               "NER_ENTITIES",
-                                               "TABLES",
-                                               "IMPORTED_REDACTIONS")
                 .stream()
                 .map(FileType::valueOf)
                 .collect(Collectors.toSet());
 
+        Set<String> oldFileSuffixes = Set.of("DOCUMENT_PAGES", "DOCUMENT_POSITION", "DOCUMENT_STRUCTURE", "DOCUMENT_TEXT");
+
         Set<FileType> uploadedFileTypes = Files.walk(folder)
                 .filter(path -> path.toFile().isFile())
                 .filter(path -> parseFileTypeFromPath(path).map(endingsToUpload::contains)
                         .orElse(false))
-                .map(filePath -> uploadFile(filePath, request))
+                .map(filePath -> uploadFile(filePath, request, true))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
                 .collect(Collectors.toUnmodifiableSet());
 
+        Files.walk(folder)
+                .filter(path -> path.toFile().isFile())
+                .filter(path -> oldFileSuffixes.contains(parseFileTypeAndExtensionFromPath(path).getFirst()))
+                .forEach(filePath -> uploadFile(filePath, request, false));
+
         Set<FileType> missingFileTypes = Sets.difference(endingsToUpload, uploadedFileTypes);
         if (!missingFileTypes.isEmpty()) {
             log.error("Folder {} is missing files of type {}",
@@ -353,16 +543,36 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
+    private static Pair<String, String> parseFileTypeAndExtensionFromPath(Path path) {
+
+        String[] parts = path.getFileName().toString().split("\\.");
+        String fileType = parts[1];
+        String extension = parts[2];
+        return Pair.of(fileType, extension);
+    }
+
+
     @SneakyThrows
-    private Optional<FileType> uploadFile(Path path, AnalyzeRequest request) {
+    private Optional<FileType> uploadFile(Path path, AnalyzeRequest request, boolean parseAsFileType) {
+
+        Optional<FileType> fileType = Optional.empty();
+        String storageId = null;
+        if (parseAsFileType) {
+
+            fileType = parseFileTypeFromPath(path);
+            if (fileType.isPresent()) {
+                storageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get());
+            }
+        } else {
+            Pair<String, String> fileTypeAndExtension = parseFileTypeAndExtensionFromPath(path);
+            storageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileTypeAndExtension.getFirst() + "." + fileTypeAndExtension.getSecond());
 
-        Optional<FileType> fileType = parseFileTypeFromPath(path);
-        if (fileType.isEmpty()) {
-            return Optional.empty();
         }
-        try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis);) {
+        if(storageId != null) {
-            storageService.storeObject(TENANT_ID, RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get()), in);
+            try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis)) {
+                storageService.storeObject(TENANT_ID, storageId, in);
 
+            }
         }
         return fileType;
     }