Marmelator/image-classification-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

tidy up repo

Browse Source
This commit is contained in:
cdietrich 2022-03-02 10:15:08 +01:00
parent 42ae5793e0
commit 372d6645d7
11 changed files with 32 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile_base
View File

@ -26,7 +26,3 @@ COPY --from=builder1 /app .
ENV PATH="/app/venv/bin:$PATH"
WORKDIR /app/service
RUN apt update --yes
RUN apt install vim --yes
RUN apt install poppler-utils --yes

2
README.md
View File

@ -21,5 +21,5 @@ docker run --rm --net=host --rm image-prediction
Shell 2
```bash
python scripts/client_mock.py --pdf_path /path/to/a/pdf
python scripts/pyinfra_mock.py --pdf_path /path/to/a/pdf
```

2
bamboo-specs/src/main/java/buildjob/PlanSpec.java
View File

@ -85,7 +85,7 @@ public class PlanSpec {
.description("Checkout default repository.")
.checkoutItems(new CheckoutItem().defaultRepository()),
new VcsCheckoutTask()
.description("Checkout detr research repository.")
.description("Checkout redai_image research repository.")
.checkoutItems(new CheckoutItem().repository("RR / redai_image").path("redai_image")),
new ScriptTask()
.description("Set config and keys.")

8
bamboo-specs/src/main/resources/scripts/sonar-scan.sh
View File

@ -10,7 +10,7 @@ python3 -m pip install --upgrade pip
echo "dev setup for unit test and coverage 💖"
pip install -e .
pip install -e incl/detr
pip install -e incl/redai_image
pip install -r requirements.txt
SERVICE_NAME=$1
@ -19,14 +19,14 @@ echo "dependency-check:aggregate"
mkdir -p reports
dependency-check --enableExperimental -f JSON -f HTML -f XML \
--disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \
--exclude "build_venv/**" --exclude "**/__pycache__/**"
--exclude "build_venv/**" --exclude "**/__pycache__/**" --exclude "bamboo-specs/**"
if [[ -z "${bamboo_repository_pr_key}" ]]
then
echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
/usr/bin/sonar-scanner/bin/sonar-scanner \
-Dsonar.projectKey=RED_$SERVICE_NAME \
-Dsonar.sources=incl/image_service/image_service,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \
-Dsonar.sources=image_prediction,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.branch.name=${bamboo_planRepository_1_branch} \
@ -39,7 +39,7 @@ else
echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
/usr/bin/sonar-scanner/bin/sonar-scanner \
-Dsonar.projectKey=RED_$SERVICE_NAME \
-Dsonar.sources=incl/image_service/image_service,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \
-Dsonar.sources=image_prediction,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.pullrequest.key=${bamboo_repository_pr_key} \

10
docker-compose.yaml
View File

@ -1,10 +0,0 @@
version: "3.3"
services:
detr-server:
image: detr-server
network_mode: "host"
read_only: true
volumes:
- tmp:/tmp:rw
volumes:
tmp:

0
image_prediction/utils/__init__.py
View File

96
image_prediction/utils/non_max_supprs.py
View File

@ -1,96 +0,0 @@
from collections import namedtuple
from itertools import starmap, combinations
from operator import attrgetter, itemgetter
from frozendict import frozendict
Rectangle = namedtuple("Rectangle", "xmin ymin xmax ymax")
def make_box(x1, y1, x2, y2):
keys = "x1", "y1", "x2", "y2"
return dict(zip(keys, [x1, y1, x2, y2]))
def compute_intersection(a, b):
a = Rectangle(*a.values())
b = Rectangle(*b.values())
dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin)
dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin)
return dx * dy if (dx >= 0) and (dy >= 0) else 0
def compute_union(a, b):
def area(box):
r = Rectangle(*box.values())
return (r.xmax - r.xmin) * (r.ymax - r.ymin)
return (area(a) + area(b)) - compute_intersection(a, b)
def compute_iou(a, b):
return compute_intersection(a, b) / compute_union(a, b)
LPBox = namedtuple("LPBox", "label proba box")
def less_likely(a, b):
return min([a, b], key=attrgetter("proba"))
def overlap_too_much(a, b, iou_thresh):
iou = compute_iou(a.box, b.box)
return iou > iou_thresh
def __greedy_non_max_supprs(lpboxes, iou_thresh=0.1):
def remove_less_likely(a, b):
try:
ll = less_likely(a, b)
current_boxes.remove(ll)
except KeyError:
pass
current_boxes = {*lpboxes}
while True:
n = len(current_boxes)
for a, b in combinations(current_boxes, r=2):
if len({a, b} & current_boxes) != 2:
continue
if overlap_too_much(a, b, iou_thresh):
remove_less_likely(a, b)
if n == len(current_boxes):
break
return current_boxes
def lpboxes_to_dict(lpboxes):
boxes = map(dict, map(attrgetter("box"), lpboxes))
classes = map(attrgetter("label"), lpboxes)
probas = map(attrgetter("proba"), lpboxes)
boxes, classes, probas = map(list, [boxes, classes, probas])
return {"bboxes": boxes, "classes": classes, "probas": probas}
def greedy_non_max_supprs(predictions):
boxes, classes, probas = itemgetter("bboxes", "classes", "probas")(predictions)
boxes = map(frozendict, boxes)
lpboxes = list(starmap(LPBox, zip(classes, probas, boxes)))
lpboxes = __greedy_non_max_supprs(lpboxes)
merged_predictions = lpboxes_to_dict(lpboxes)
predictions.update(merged_predictions)
return predictions

20
image_prediction/utils/stream.py
View File

@ -1,20 +0,0 @@
from itertools import takewhile, starmap, islice, repeat
from operator import truth
from pdf2image import pdf2image
def chunk_iterable(iterable, n):
return takewhile(truth, map(tuple, starmap(islice, repeat((iter(iterable), n)))))
def get_page_count(pdf):
return pdf2image.pdfinfo_from_bytes(pdf)["Pages"]
def stream_pages(pdf):
def page_to_image(idx):
return pdf2image.convert_from_bytes(pdf, first_page=idx, last_page=idx + 1)[0]
page_count = get_page_count(pdf)
return map(page_to_image, range(page_count))

58
scripts/client_mock.py
View File

@ -1,58 +0,0 @@
import argparse
import json
from operator import itemgetter
import pdf2image
import requests
from PIL import ImageDraw, ImageFont
def draw_coco_box(draw: ImageDraw.Draw, bbox, klass, proba):
x1, y1, x2, y2 = itemgetter("x1", "y1", "x2", "y2")(bbox)
draw.rectangle(((x1, y1), (x2, y2)), outline="red")
fnt = ImageFont.truetype("Pillow/Tests/fonts/FreeMono.ttf", 30)
draw.text((x1, y2), text=f"{klass}: {proba:.2f}", fill=(0, 0, 0, 100), font=fnt)
def draw_coco_boxes(image, bboxes, classes, probas):
draw = ImageDraw.Draw(image)
for bbox, klass, proba in zip(bboxes, classes, probas):
draw_coco_box(draw, bbox, klass, proba)
return image
def annotate(pdf_path, predictions):
pages = pdf2image.convert_from_path(pdf_path)
for prd in predictions:
page_idx, boxes, classes, probas = itemgetter("page_idx", "bboxes", "classes", "probas")(prd)
page = pages[page_idx]
image = draw_coco_boxes(page, boxes, classes, probas)
image.save(f"/tmp/serv_out/{page_idx}.png")
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--pdf_path", required=True)
args = parser.parse_args()
return args
def main(args):
response = requests.post("http://127.0.0.1:5000", data=open(args.pdf_path, "rb"))
response.raise_for_status()
predictions = response.json()
print(json.dumps(predictions, indent=2))
annotate(args.pdf_path, predictions)
if __name__ == "__main__":
args = parse_args()
main(args)

35
scripts/flask_test.py
View File

@ -1,35 +0,0 @@
import argparse
from PIL import Image
from flask import Flask, request, jsonify
from pathlib import Path
app = Flask(__name__)
@app.before_first_request
def init():
from image_prediction.predictor import Predictor
global PRED
PRED = Predictor(args.resume)
@app.route("/", methods=["GET", "POST"])
def predict_request():
if request.method == "POST":
image_folder_path = request.form.get("image_folder_path")
images = list(map(Image.open, Path(image_folder_path).glob("*.png")))
results = PRED.predict(images, format_output=True)
for result in results:
return jsonify(result)
if request.method == "GET":
return "Not implemented"
parser = argparse.ArgumentParser()
parser.add_argument("--resume", required=True)
args = parser.parse_args()
app.run()

26
scripts/pyinfra_mock.py Normal file
View File

@ -0,0 +1,26 @@
import argparse
import json
import requests
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--pdf_path", required=True)
args = parser.parse_args()
return args
def main(args):
response = requests.post("http://127.0.0.1:5000", data=open(args.pdf_path, "rb"))
response.raise_for_status()
predictions = response.json()
print(json.dumps(predictions, indent=2))
if __name__ == "__main__":
args = parse_args()
main(args)