109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
import os
|
|
from _operator import itemgetter
|
|
|
|
from funcy import project
|
|
|
|
from pyinfra.config import CONFIG
|
|
|
|
|
|
class FileDescriptorManager:
|
|
def __init__(self, bucket_name, operation2file_patterns: dict = None):
|
|
self.bucket_name = bucket_name
|
|
self.operation2file_patterns = operation2file_patterns or self.get_default_operation2file_patterns()
|
|
|
|
@staticmethod
|
|
def get_default_operation2file_patterns():
|
|
return {"default": {"input": {"subdir": "", "extension": ".in"}, "output": {"subdir": "", "extension": ".out"}}}
|
|
|
|
def get_input_object_name(self, queue_item_body: dict):
|
|
return self.get_object_name(queue_item_body, end="input")
|
|
|
|
def get_output_object_name(self, queue_item_body: dict):
|
|
return self.get_object_name(queue_item_body, end="output")
|
|
|
|
def get_object_name(self, queue_item_body: dict, end):
|
|
|
|
file_descriptor = self.build_file_descriptor(queue_item_body, end=end)
|
|
file_descriptor["pages"] = [queue_item_body.get("id", 0)]
|
|
|
|
object_name = self.__build_matcher(file_descriptor)
|
|
|
|
return object_name
|
|
|
|
def __build_matcher(self, file_descriptor):
|
|
def make_filename(file_id, subdir, suffix):
|
|
return os.path.join(file_id, subdir, suffix) if subdir else f"{file_id}.{suffix}"
|
|
|
|
dossier_id, file_id, subdir, pages, extension = itemgetter(
|
|
"dossierId", "fileId", "subdir", "pages", "extension"
|
|
)(file_descriptor)
|
|
|
|
matcher = os.path.join(
|
|
dossier_id, make_filename(file_id, subdir, self.__build_page_regex(pages, subdir) + extension)
|
|
)
|
|
|
|
return matcher
|
|
|
|
@staticmethod
|
|
def __build_page_regex(pages, subdir):
|
|
if len(pages) > 1 and subdir:
|
|
page_re = "id:(" + "|".join(map(str, pages)) + ")."
|
|
elif not pages and subdir:
|
|
page_re = r"id:\d+."
|
|
elif len(pages) == 1 and subdir:
|
|
page_re = f"id:{pages[0]}."
|
|
else:
|
|
page_re = ""
|
|
|
|
return page_re
|
|
|
|
def build_file_descriptor(self, queue_item_body, end="input"):
|
|
# FIXME: reference to config is misplaced here. Needs to be abstracted in some way.
|
|
operation = queue_item_body.get("operation", CONFIG.service.operation)
|
|
|
|
file_pattern = self.operation2file_patterns[operation][end]
|
|
|
|
file_descriptor = {
|
|
**project(queue_item_body, ["dossierId", "fileId", "pages"]),
|
|
"pages": queue_item_body.get("pages", []),
|
|
"extension": file_pattern["extension"],
|
|
"subdir": file_pattern["subdir"],
|
|
}
|
|
return file_descriptor
|
|
|
|
def build_input_matcher(self, queue_item_body):
|
|
return self.build_matcher(queue_item_body, end="input")
|
|
|
|
def build_output_matcher(self, queue_item_body):
|
|
return self.build_matcher(queue_item_body, end="output")
|
|
|
|
def build_matcher(self, queue_item_body, end):
|
|
file_descriptor = self.build_file_descriptor(queue_item_body, end=end)
|
|
return self.__build_matcher(file_descriptor)
|
|
|
|
def get_input_object_descriptor(self, queue_item_body):
|
|
return self.get_object_descriptor(queue_item_body, end="input")
|
|
|
|
def get_output_object_descriptor(self, storage_upload_info):
|
|
return self.get_object_descriptor(storage_upload_info, end="output")
|
|
|
|
def get_object_descriptor(self, queue_item_body, end):
|
|
return {
|
|
"bucket_name": self.bucket_name,
|
|
"object_name": self.get_object_name(queue_item_body, end=end),
|
|
}
|
|
|
|
@staticmethod
|
|
def build_storage_upload_info(analysis_payload, request_metadata):
|
|
storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)}
|
|
return storage_upload_info
|
|
|
|
def get_path_prefix(self, queue_item_body):
|
|
|
|
prefix = "/".join(
|
|
itemgetter("dossierId", "fileId")(
|
|
self.build_file_descriptor(queue_item_body, end="input"),
|
|
)
|
|
)
|
|
return prefix
|