From 372d6645d75d8e6c6fc00aa150a5bb8a75650146 Mon Sep 17 00:00:00 2001 From: cdietrich Date: Wed, 2 Mar 2022 10:15:08 +0100 Subject: [PATCH] tidy up repo --- Dockerfile_base | 4 - README.md | 2 +- .../src/main/java/buildjob/PlanSpec.java | 2 +- .../src/main/resources/scripts/sonar-scan.sh | 8 +- docker-compose.yaml | 10 -- image_prediction/utils/__init__.py | 0 image_prediction/utils/non_max_supprs.py | 96 ------------------- image_prediction/utils/stream.py | 20 ---- scripts/client_mock.py | 58 ----------- scripts/flask_test.py | 35 ------- scripts/pyinfra_mock.py | 26 +++++ 11 files changed, 32 insertions(+), 229 deletions(-) delete mode 100644 docker-compose.yaml delete mode 100644 image_prediction/utils/__init__.py delete mode 100644 image_prediction/utils/non_max_supprs.py delete mode 100644 image_prediction/utils/stream.py delete mode 100644 scripts/client_mock.py delete mode 100644 scripts/flask_test.py create mode 100644 scripts/pyinfra_mock.py diff --git a/Dockerfile_base b/Dockerfile_base index c8decb7..81639d5 100644 --- a/Dockerfile_base +++ b/Dockerfile_base @@ -26,7 +26,3 @@ COPY --from=builder1 /app . ENV PATH="/app/venv/bin:$PATH" WORKDIR /app/service - -RUN apt update --yes -RUN apt install vim --yes -RUN apt install poppler-utils --yes diff --git a/README.md b/README.md index 41e2f56..f913627 100644 --- a/README.md +++ b/README.md @@ -21,5 +21,5 @@ docker run --rm --net=host --rm image-prediction Shell 2 ```bash -python scripts/client_mock.py --pdf_path /path/to/a/pdf +python scripts/pyinfra_mock.py --pdf_path /path/to/a/pdf ``` diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java index adba0f2..4a4d308 100644 --- a/bamboo-specs/src/main/java/buildjob/PlanSpec.java +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -85,7 +85,7 @@ public class PlanSpec { .description("Checkout default repository.") .checkoutItems(new CheckoutItem().defaultRepository()), new VcsCheckoutTask() - .description("Checkout detr research repository.") + .description("Checkout redai_image research repository.") .checkoutItems(new CheckoutItem().repository("RR / redai_image").path("redai_image")), new ScriptTask() .description("Set config and keys.") diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh index 693f216..febc142 100755 --- a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh +++ b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh @@ -10,7 +10,7 @@ python3 -m pip install --upgrade pip echo "dev setup for unit test and coverage 💖" pip install -e . -pip install -e incl/detr +pip install -e incl/redai_image pip install -r requirements.txt SERVICE_NAME=$1 @@ -19,14 +19,14 @@ echo "dependency-check:aggregate" mkdir -p reports dependency-check --enableExperimental -f JSON -f HTML -f XML \ --disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \ - --exclude "build_venv/**" --exclude "**/__pycache__/**" + --exclude "build_venv/**" --exclude "**/__pycache__/**" --exclude "bamboo-specs/**" if [[ -z "${bamboo_repository_pr_key}" ]] then echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}" /usr/bin/sonar-scanner/bin/sonar-scanner \ -Dsonar.projectKey=RED_$SERVICE_NAME \ - -Dsonar.sources=incl/image_service/image_service,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \ + -Dsonar.sources=image_prediction,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \ -Dsonar.host.url=https://sonarqube.iqser.com \ -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ -Dsonar.branch.name=${bamboo_planRepository_1_branch} \ @@ -39,7 +39,7 @@ else echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}" /usr/bin/sonar-scanner/bin/sonar-scanner \ -Dsonar.projectKey=RED_$SERVICE_NAME \ - -Dsonar.sources=incl/image_service/image_service,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \ + -Dsonar.sources=image_prediction,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \ -Dsonar.host.url=https://sonarqube.iqser.com \ -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 0afb54b..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3.3" -services: - detr-server: - image: detr-server - network_mode: "host" - read_only: true - volumes: - - tmp:/tmp:rw -volumes: - tmp: diff --git a/image_prediction/utils/__init__.py b/image_prediction/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/image_prediction/utils/non_max_supprs.py b/image_prediction/utils/non_max_supprs.py deleted file mode 100644 index f38a63e..0000000 --- a/image_prediction/utils/non_max_supprs.py +++ /dev/null @@ -1,96 +0,0 @@ -from collections import namedtuple -from itertools import starmap, combinations -from operator import attrgetter, itemgetter - -from frozendict import frozendict - -Rectangle = namedtuple("Rectangle", "xmin ymin xmax ymax") - - -def make_box(x1, y1, x2, y2): - keys = "x1", "y1", "x2", "y2" - return dict(zip(keys, [x1, y1, x2, y2])) - - -def compute_intersection(a, b): - - a = Rectangle(*a.values()) - b = Rectangle(*b.values()) - - dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin) - dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin) - - return dx * dy if (dx >= 0) and (dy >= 0) else 0 - - -def compute_union(a, b): - def area(box): - r = Rectangle(*box.values()) - return (r.xmax - r.xmin) * (r.ymax - r.ymin) - - return (area(a) + area(b)) - compute_intersection(a, b) - - -def compute_iou(a, b): - return compute_intersection(a, b) / compute_union(a, b) - - -LPBox = namedtuple("LPBox", "label proba box") - - -def less_likely(a, b): - return min([a, b], key=attrgetter("proba")) - - -def overlap_too_much(a, b, iou_thresh): - iou = compute_iou(a.box, b.box) - return iou > iou_thresh - - -def __greedy_non_max_supprs(lpboxes, iou_thresh=0.1): - def remove_less_likely(a, b): - try: - ll = less_likely(a, b) - current_boxes.remove(ll) - except KeyError: - pass - - current_boxes = {*lpboxes} - - while True: - n = len(current_boxes) - for a, b in combinations(current_boxes, r=2): - if len({a, b} & current_boxes) != 2: - continue - if overlap_too_much(a, b, iou_thresh): - remove_less_likely(a, b) - - if n == len(current_boxes): - break - - return current_boxes - - -def lpboxes_to_dict(lpboxes): - - boxes = map(dict, map(attrgetter("box"), lpboxes)) - classes = map(attrgetter("label"), lpboxes) - probas = map(attrgetter("proba"), lpboxes) - - boxes, classes, probas = map(list, [boxes, classes, probas]) - - return {"bboxes": boxes, "classes": classes, "probas": probas} - - -def greedy_non_max_supprs(predictions): - - boxes, classes, probas = itemgetter("bboxes", "classes", "probas")(predictions) - boxes = map(frozendict, boxes) - lpboxes = list(starmap(LPBox, zip(classes, probas, boxes))) - - lpboxes = __greedy_non_max_supprs(lpboxes) - - merged_predictions = lpboxes_to_dict(lpboxes) - predictions.update(merged_predictions) - - return predictions diff --git a/image_prediction/utils/stream.py b/image_prediction/utils/stream.py deleted file mode 100644 index d9948a3..0000000 --- a/image_prediction/utils/stream.py +++ /dev/null @@ -1,20 +0,0 @@ -from itertools import takewhile, starmap, islice, repeat -from operator import truth - -from pdf2image import pdf2image - - -def chunk_iterable(iterable, n): - return takewhile(truth, map(tuple, starmap(islice, repeat((iter(iterable), n))))) - - -def get_page_count(pdf): - return pdf2image.pdfinfo_from_bytes(pdf)["Pages"] - - -def stream_pages(pdf): - def page_to_image(idx): - return pdf2image.convert_from_bytes(pdf, first_page=idx, last_page=idx + 1)[0] - - page_count = get_page_count(pdf) - return map(page_to_image, range(page_count)) diff --git a/scripts/client_mock.py b/scripts/client_mock.py deleted file mode 100644 index 7d26000..0000000 --- a/scripts/client_mock.py +++ /dev/null @@ -1,58 +0,0 @@ -import argparse -import json -from operator import itemgetter - -import pdf2image -import requests -from PIL import ImageDraw, ImageFont - - -def draw_coco_box(draw: ImageDraw.Draw, bbox, klass, proba): - x1, y1, x2, y2 = itemgetter("x1", "y1", "x2", "y2")(bbox) - draw.rectangle(((x1, y1), (x2, y2)), outline="red") - - fnt = ImageFont.truetype("Pillow/Tests/fonts/FreeMono.ttf", 30) - - draw.text((x1, y2), text=f"{klass}: {proba:.2f}", fill=(0, 0, 0, 100), font=fnt) - - -def draw_coco_boxes(image, bboxes, classes, probas): - - draw = ImageDraw.Draw(image) - for bbox, klass, proba in zip(bboxes, classes, probas): - draw_coco_box(draw, bbox, klass, proba) - - return image - - -def annotate(pdf_path, predictions): - pages = pdf2image.convert_from_path(pdf_path) - - for prd in predictions: - page_idx, boxes, classes, probas = itemgetter("page_idx", "bboxes", "classes", "probas")(prd) - page = pages[page_idx] - image = draw_coco_boxes(page, boxes, classes, probas) - image.save(f"/tmp/serv_out/{page_idx}.png") - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--pdf_path", required=True) - args = parser.parse_args() - - return args - - -def main(args): - - response = requests.post("http://127.0.0.1:5000", data=open(args.pdf_path, "rb")) - response.raise_for_status() - predictions = response.json() - - print(json.dumps(predictions, indent=2)) - annotate(args.pdf_path, predictions) - - -if __name__ == "__main__": - args = parse_args() - main(args) diff --git a/scripts/flask_test.py b/scripts/flask_test.py deleted file mode 100644 index ba95c12..0000000 --- a/scripts/flask_test.py +++ /dev/null @@ -1,35 +0,0 @@ -import argparse - -from PIL import Image -from flask import Flask, request, jsonify -from pathlib import Path - -app = Flask(__name__) - - -@app.before_first_request -def init(): - from image_prediction.predictor import Predictor - - global PRED - - PRED = Predictor(args.resume) - - -@app.route("/", methods=["GET", "POST"]) -def predict_request(): - if request.method == "POST": - image_folder_path = request.form.get("image_folder_path") - images = list(map(Image.open, Path(image_folder_path).glob("*.png"))) - results = PRED.predict(images, format_output=True) - for result in results: - return jsonify(result) - if request.method == "GET": - return "Not implemented" - - -parser = argparse.ArgumentParser() -parser.add_argument("--resume", required=True) -args = parser.parse_args() - -app.run() diff --git a/scripts/pyinfra_mock.py b/scripts/pyinfra_mock.py new file mode 100644 index 0000000..fec12e9 --- /dev/null +++ b/scripts/pyinfra_mock.py @@ -0,0 +1,26 @@ +import argparse +import json + +import requests + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--pdf_path", required=True) + args = parser.parse_args() + + return args + + +def main(args): + + response = requests.post("http://127.0.0.1:5000", data=open(args.pdf_path, "rb")) + response.raise_for_status() + predictions = response.json() + + print(json.dumps(predictions, indent=2)) + + +if __name__ == "__main__": + args = parse_args() + main(args)