From 778e2fa858236562d1321d59151834f1411ae633 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Thu, 18 Jul 2024 12:52:04 +0200 Subject: [PATCH] evaluation whether ids stay the same when switching from json to protobuf --- .../storage/RedactionStorageService.java | 4 + .../v1/server/AnalysisEnd2EndTest.java | 258 ++++++++++++++++-- 2 files changed, 238 insertions(+), 24 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 5da87610..3163a57c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -313,6 +313,10 @@ public class RedactionStorageService { return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); } + public static String getStorageId(String userId, String dossierId, String filename) { + + return userId + "/" + dossierId + "/" + filename; + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index 7d09a30c..d6445cbe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -11,6 +11,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.nio.file.FileVisitOption; @@ -41,6 +42,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.Import; import org.springframework.core.io.ClassPathResource; import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.data.util.Pair; import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.core.JsonProcessingException; @@ -52,6 +54,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; @@ -68,12 +71,24 @@ import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils; import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; +import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor; import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantProvider; import com.knecon.fforesight.tenantcommons.model.MongoDBConnection; @@ -142,12 +157,113 @@ import lombok.extern.slf4j.Slf4j; @Autowired RedactionStorageService redactionStorageService; + private static EntryDataProto.EntryData convertEntryData(DocumentStructure.EntryData oldEntryData) { + EntryDataProto.EntryData.Builder builder = EntryDataProto.EntryData.newBuilder(); + + builder.setType(NodeTypeProto.NodeType.valueOf(oldEntryData.getType().name())); + builder.addAllTreeId(Arrays.stream(oldEntryData.getTreeId()).boxed().collect(Collectors.toList())); + builder.addAllAtomicBlockIds(Arrays.asList(oldEntryData.getAtomicBlockIds())); + builder.addAllPageNumbers(Arrays.asList(oldEntryData.getPageNumbers())); + + builder.putAllProperties(oldEntryData.getProperties()); + + // Recursively converting children + if (oldEntryData.getChildren() != null) { + oldEntryData.getChildren().forEach(child -> builder.addChildren(convertEntryData(child))); + } + + return builder.build(); + } + + private static DocumentStructureProto.DocumentStructure convertDocumentStructure(DocumentStructure oldStructure) { + DocumentStructureProto.DocumentStructure.Builder newBuilder = DocumentStructureProto.DocumentStructure.newBuilder(); + + if (oldStructure.getRoot() != null) { + newBuilder.setRoot(convertEntryData(oldStructure.getRoot())); + } + + return newBuilder.build(); + } + + private static DocumentPageProto.DocumentPage convertDocumentPage(DocumentPage oldPage) { + return DocumentPageProto.DocumentPage.newBuilder() + .setNumber(oldPage.getNumber()) + .setHeight(oldPage.getHeight()) + .setWidth(oldPage.getWidth()) + .setRotation(oldPage.getRotation()) + .build(); + } + + private static DocumentPageProto.AllDocumentPages convertAllDocumentPages(DocumentPage[] oldPages) { + DocumentPageProto.AllDocumentPages.Builder allPagesBuilder = DocumentPageProto.AllDocumentPages.newBuilder(); + + for (DocumentPage oldPage : oldPages) { + DocumentPageProto.DocumentPage newPage = convertDocumentPage(oldPage); + allPagesBuilder.addDocumentPages(newPage); + } + + return allPagesBuilder.build(); + } + + private static DocumentPositionDataProto.DocumentPositionData convertPositionData(DocumentPositionData oldData) { + DocumentPositionDataProto.DocumentPositionData.Builder builder = DocumentPositionDataProto.DocumentPositionData.newBuilder() + .setId(oldData.getId()) + .addAllStringIdxToPositionIdx(Arrays.stream(oldData.getStringIdxToPositionIdx()).boxed().collect(Collectors.toList())); + + // Convert positions array to Position messages + for (float[] pos : oldData.getPositions()) { + DocumentPositionDataProto.DocumentPositionData.Position position = DocumentPositionDataProto.DocumentPositionData.Position.newBuilder() + .setX(pos[0]) + .setY(pos[1]) + .setWidth(pos[2]) + .setHeight(pos[3]) + .build(); + builder.addPositions(position); + } + + return builder.build(); + } + + private static DocumentPositionDataProto.AllDocumentPositionData convertAllPositionData(DocumentPositionData[] oldDataList) { + DocumentPositionDataProto.AllDocumentPositionData.Builder allDataBuilder = DocumentPositionDataProto.AllDocumentPositionData.newBuilder(); + + for (DocumentPositionData oldData : oldDataList) { + allDataBuilder.addDocumentPositionData(convertPositionData(oldData)); + } + + return allDataBuilder.build(); + } + + public static DocumentTextDataProto.DocumentTextData convertTextData(DocumentTextData oldData) { + DocumentTextDataProto.DocumentTextData.Builder builder = DocumentTextDataProto.DocumentTextData.newBuilder() + .setId(oldData.getId()) + .setPage(oldData.getPage()) + .setSearchText(oldData.getSearchText()) + .setNumberOnPage(oldData.getNumberOnPage()) + .setStart(oldData.getStart()) + .setEnd(oldData.getEnd()) + .addAllLineBreaks(Arrays.stream(oldData.getLineBreaks()).boxed().collect(Collectors.toList())); + + return builder.build(); + } + + public static DocumentTextDataProto.AllDocumentTextData convertAllTextData(DocumentTextData[] oldDataList) { + DocumentTextDataProto.AllDocumentTextData.Builder allDataBuilder = DocumentTextDataProto.AllDocumentTextData.newBuilder(); + + for (DocumentTextData oldData : oldDataList) { + allDataBuilder.addDocumentTextData(convertTextData(oldData)); + } + + return allDataBuilder.build(); + } + + @Test @SneakyThrows public void runAnalysisEnd2End() { - String folder = "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/files_end2end/file_large2"; // Should contain all files from minio directly, still zipped. Can contain multiple files. + String folder = "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/files_end2end/file_large"; // Should contain all files from minio directly, still zipped. Can contain multiple files. Path absoluteFolderPath; if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path @@ -169,15 +285,78 @@ import lombok.extern.slf4j.Slf4j; ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); - //analyzeService.analyze(analyzeRequest); - //storeEntityLogAsTempFile(analyzeRequest, objectMapper); + String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + FileType.DOCUMENT_STRUCTURE.name() + ".json"); + String newDocumentStructureStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_STRUCTURE); + try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentStructureStorageId)) { + DocumentStructure oldDocumentStructure = objectMapper.readValue(inputStream, DocumentStructure.class); + DocumentStructureProto.DocumentStructure newDocumentStructure = convertDocumentStructure(oldDocumentStructure); + storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentStructureStorageId, newDocumentStructure); + storageService.deleteObject(TenantContext.getTenantId(), oldDocumentStructureStorageId); + } catch (StorageObjectDoesNotExist e) { + log.info("No document structure found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + } - restoreEntityLogFromTempFile(objectMapper, analyzeRequest); + String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_PAGES.name() + ".json"); + String newDocumentPagesStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_PAGES); + try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentPagesStorageId)) { + DocumentPage[] oldDocumentPages = objectMapper.readValue(inputStream, DocumentPage[].class); + DocumentPageProto.AllDocumentPages newDocumentPages = convertAllDocumentPages(oldDocumentPages); + storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPagesStorageId, newDocumentPages); + storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPagesStorageId); + } catch (StorageObjectDoesNotExist e) { + log.info("No document pages found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + } - addManualRedactionEntryToFile(analyzeRequest); + String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + FileType.DOCUMENT_POSITION.name() + ".json"); + String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_POSITION); + try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentPositionDataStorageId)) { + DocumentPositionData[] oldDocumentPositionData = objectMapper.readValue(inputStream, DocumentPositionData[].class); + DocumentPositionDataProto.AllDocumentPositionData newDocumentPositionData = convertAllPositionData(oldDocumentPositionData); + storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId, newDocumentPositionData); + storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId); + } catch (StorageObjectDoesNotExist e) { + log.info("No document position data found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + } + + String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + FileType.DOCUMENT_TEXT.name() + ".json"); + String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_TEXT); + try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentTextDataStorageId)) { + DocumentTextData[] oldDocumentTextData = objectMapper.readValue(inputStream, DocumentTextData[].class); + DocumentTextDataProto.AllDocumentTextData newDocumentTextData = convertAllTextData(oldDocumentTextData); + storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextData); + storageService.deleteObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId); + } catch (StorageObjectDoesNotExist e) { + log.info("No document text data found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + } analyzeService.reanalyze(analyzeRequest); + storeEntityLogAsTempFile(analyzeRequest, objectMapper); + + EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); times.add(System.currentTimeMillis() - start); + + EntityLog otherEntityLog = getEntityLogFromTemp("/tmp/entityLog2.ser", objectMapper); + + List entryIds = entityLog.getEntityLogEntry() + .stream() + .map(EntityLogEntry::getId) + .toList(); + List otherEntryIds = otherEntityLog.getEntityLogEntry() + .stream() + .map(EntityLogEntry::getId) + .toList(); + if(entryIds.containsAll(otherEntryIds) && otherEntryIds.containsAll(entryIds)) { + System.out.println("We won"); + } else { + System.out.println("We lost"); + } + } System.out.println("times in ms for each analyze run: " + times); } @@ -202,7 +381,7 @@ import lombok.extern.slf4j.Slf4j; } - private void restoreEntityLogFromTempFile(ObjectMapper objectMapper, AnalyzeRequest analyzeRequest) throws JsonProcessingException { + private void restoreEntityLogFromTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException { //Deserialize entityLog from a file String entityLogString = null; try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) { @@ -214,6 +393,17 @@ import lombok.extern.slf4j.Slf4j; redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); } + private EntityLog getEntityLogFromTemp(String fileName, ObjectMapper objectMapper) throws JsonProcessingException { + //Deserialize entityLog from a file + String entityLogString = null; + try (FileInputStream fileIn = new FileInputStream(fileName); ObjectInputStream in = new ObjectInputStream(fileIn)) { + entityLogString = (String) in.readObject(); + } catch (IOException | ClassNotFoundException e) { + e.printStackTrace(); + } + return objectMapper.readValue(entityLogString, EntityLog.class); + } + private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException { @@ -222,8 +412,7 @@ import lombok.extern.slf4j.Slf4j; // Serialize String jsonString = objectMapper.writeValueAsString(entityLog); // Serialize entityLog to a file - try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); - ObjectOutputStream out = new ObjectOutputStream(fileOut)) { + try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); ObjectOutputStream out = new ObjectOutputStream(fileOut)) { out.writeObject(jsonString); } catch (IOException e) { e.printStackTrace(); @@ -274,6 +463,8 @@ import lombok.extern.slf4j.Slf4j; } + + @SneakyThrows private List prepareStorageForFolder(Path folder) { @@ -307,28 +498,27 @@ import lombok.extern.slf4j.Slf4j; request.setManualRedactions(new ManualRedactions()); } - Set endingsToUpload = Set.of("ORIGIN", - "DOCUMENT_PAGES", - "DOCUMENT_POSITION", - "DOCUMENT_STRUCTURE", - "DOCUMENT_TEXT", - "IMAGE_INFO", - "NER_ENTITIES", - "TABLES", - "IMPORTED_REDACTIONS") + Set endingsToUpload = Set.of("ORIGIN", "IMAGE_INFO", "NER_ENTITIES", "TABLES", "IMPORTED_REDACTIONS") .stream() .map(FileType::valueOf) .collect(Collectors.toSet()); + Set oldFileSuffixes = Set.of("DOCUMENT_PAGES", "DOCUMENT_POSITION", "DOCUMENT_STRUCTURE", "DOCUMENT_TEXT"); + Set uploadedFileTypes = Files.walk(folder) .filter(path -> path.toFile().isFile()) .filter(path -> parseFileTypeFromPath(path).map(endingsToUpload::contains) .orElse(false)) - .map(filePath -> uploadFile(filePath, request)) + .map(filePath -> uploadFile(filePath, request, true)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toUnmodifiableSet()); + Files.walk(folder) + .filter(path -> path.toFile().isFile()) + .filter(path -> oldFileSuffixes.contains(parseFileTypeAndExtensionFromPath(path).getFirst())) + .forEach(filePath -> uploadFile(filePath, request, false)); + Set missingFileTypes = Sets.difference(endingsToUpload, uploadedFileTypes); if (!missingFileTypes.isEmpty()) { log.error("Folder {} is missing files of type {}", @@ -353,16 +543,36 @@ import lombok.extern.slf4j.Slf4j; } + private static Pair parseFileTypeAndExtensionFromPath(Path path) { + + String[] parts = path.getFileName().toString().split("\\."); + String fileType = parts[1]; + String extension = parts[2]; + return Pair.of(fileType, extension); + } + + @SneakyThrows - private Optional uploadFile(Path path, AnalyzeRequest request) { + private Optional uploadFile(Path path, AnalyzeRequest request, boolean parseAsFileType) { + + Optional fileType = Optional.empty(); + String storageId = null; + if (parseAsFileType) { + + fileType = parseFileTypeFromPath(path); + if (fileType.isPresent()) { + storageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get()); + } + } else { + Pair fileTypeAndExtension = parseFileTypeAndExtensionFromPath(path); + storageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileTypeAndExtension.getFirst() + "." + fileTypeAndExtension.getSecond()); - Optional fileType = parseFileTypeFromPath(path); - if (fileType.isEmpty()) { - return Optional.empty(); } - try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis);) { - storageService.storeObject(TENANT_ID, RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get()), in); + if(storageId != null) { + try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis)) { + storageService.storeObject(TENANT_ID, storageId, in); + } } return fileType; }