send images as raw binary

src-tauri/Cargo.toml

@@ -28,10 +28,13 @@ tauri-plugin-dialog = "2"
 uuid = { version = "1.12.0", features = ["v4"] }
 regex = "1.10.3"
 lazy_static = "1.4.0"
-fax = "0.2"
 base64 = "0.21"
 image = { version = "0.25.5", features = ["jpeg"] }
 pdf_render = { path = "../../pdf-render/render" }
 pathfinder_rasterize = { git = "https://github.com/s3bk/pathfinder_rasterizer" }
 pathfinder_geometry = { git = "https://github.com/servo/pathfinder" }
-tauri-plugin-websocket = "2"
+show-image = {version = "0.14.0", features = ["image"] }
+
+[[example]]
+name = "render_test"
+path = "examples/render.rs"

src-tauri/capabilities/default.json

@@ -15,7 +15,6 @@
     "core:window:allow-close",
     "core:window:allow-minimize",
     "core:window:allow-toggle-maximize",
-    "core:window:allow-internal-toggle-maximize",
+    "core:window:allow-internal-toggle-maximize"
-    "websocket:default"
   ]
 }

src-tauri/src/lib.rs

@@ -1,17 +1,13 @@
-mod render;
+extern crate pdf;
+pub mod render;
 mod retrieval;
 
 #[cfg(test)]
 mod tests;
 
-extern crate pdf;
-
 use crate::pdf::object::Resolve;
 use crate::render::Renderer;
-use base64;
+use image::DynamicImage;
-use base64::prelude::BASE64_STANDARD;
-use base64::Engine;
-use image::{ImageFormat, RgbImage};
 use lazy_static::lazy_static;
 use pdf::file::{File, FileOptions, NoLog, ObjectCache, StreamCache};
 use pdf::object::{Object, PlainRef};
@@ -21,10 +17,10 @@ use regex::Regex;
 use retrieval::{get_prim_by_path_with_file, get_stream_data_by_path_with_file};
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, VecDeque};
-use std::io::Cursor;
 use std::ops::DerefMut;
 use std::path::Path;
 use std::sync::{Arc, RwLock};
+use tauri::ipc::{InvokeResponseBody, Response};
 use tauri::{Manager, State};
 use uuid::Uuid;
 
@@ -216,16 +212,33 @@ fn get_stream_data_as_string(
     Ok(String::from_utf8_lossy(&data).into_owned())
 }
 
-fn encode_b64(img: RgbImage) -> Result<String, String> {
+fn format_img_as_response(img: DynamicImage) -> Result<Response, String> {
-    let mut writer = Cursor::new(Vec::new());
+    use std::mem;
-    match img.write_to(&mut writer, ImageFormat::Jpeg) {
+
-        Ok(_) => Ok(format!(
+    let w: u32 = img.width();
-            "data:image/{};base64,{}",
+    let h: u32 = img.height();
-            "jpeg",
+    let mut data: Vec<u8> = img.into_rgba8().into_raw();
-            BASE64_STANDARD.encode(writer.into_inner())
+
-        )),
+    let w_bytes = w.to_le_bytes(); // or to_be_bytes() depending on endianness
-        Err(e) => Err(e.to_string()),
+    let h_bytes = h.to_le_bytes();
-    }
+
+    let old_capacity = data.capacity();
+
+    let ptr = data.as_mut_ptr();
+    let len = data.len();
+    mem::forget(data);
+
+    // Create a new Vec with the two u32s in front
+    let merged = unsafe {
+        let mut new_vec = Vec::from_raw_parts(ptr, len, old_capacity);
+
+        // Insert u32 bytes at the front
+        new_vec.splice(0..0, w_bytes.iter().copied());
+        new_vec.splice(4..4, h_bytes.iter().copied());
+
+        new_vec
+    };
+    Ok(Response::new(InvokeResponseBody::from(merged)))
 }
 
 #[tauri::command]
@@ -233,12 +246,10 @@ async fn get_stream_data_as_image(
     id: &str,
     path: &str,
     session: State<'_, Session>,
-) -> Result<String, String> {
+) -> Result<Response, String> {
-    use base64::prelude::*;
-
     let file = session.get_file(id)?;
     let img = retrieval::get_image_by_path(path, &file.cos_file)?;
-    encode_b64(img.into_rgb8())
+    format_img_as_response(img)
 }
 
 #[tauri::command]
@@ -246,11 +257,12 @@ async fn get_page_by_num(
     id: &str,
     num: u32,
     session: State<'_, Session>,
-) -> Result<String, String> {
+) -> Result<Response, String> {
     let file = session.get_file(id)?;
     let mut renderer = Renderer::new(&file.cos_file, 150);
     let img = renderer.render(num)?;
-    encode_b64(img.into_rgb8())
+
+    format_img_as_response(img)
 }
 
 #[tauri::command]
@@ -705,11 +717,11 @@ fn get_xref_table_model_with_file(file: &CosFile) -> Result<XRefTableModel, Stri
     })
 }
 
-struct Session {
+pub struct Session {
     files: RwLock<HashMap<String, Arc<SessionFile>>>,
 }
 
-struct SessionFile {
+pub struct SessionFile {
     pdf_file: PdfFile,
     cos_file: CosFile,
 }
@@ -770,7 +782,6 @@ impl Session {
 #[cfg_attr(mobile, tauri::mobile_entry_point)]
 pub fn run() {
     tauri::Builder::default()
-        .plugin(tauri_plugin_websocket::init())
         .plugin(tauri_plugin_dialog::init())
         .plugin(tauri_plugin_fs::init())
         .plugin(tauri_plugin_opener::init())

src-tauri/src/render.rs

@@ -6,6 +6,7 @@ use image::{DynamicImage, RgbaImage};
 use pathfinder_geometry::transform2d::Transform2F;
 use pathfinder_rasterize::Rasterizer;
 use pdf_render::{render_page, Cache, SceneBackend};
+use show_image::{create_window, run_context};
 
 pub struct Renderer<'a> {
     file: &'a CosFile,

src-tauri/src/retrieval.rs

@@ -5,14 +5,15 @@ use std::{
     sync::Arc,
 };
 
-use image::{DynamicImage, GrayImage, RgbImage};
+use image::{DynamicImage, GrayImage, RgbImage, RgbaImage};
 use pdf::{
     enc::StreamFilter,
     file::FileOptions,
     object::{ColorSpace, ImageXObject, ObjectWrite, PlainRef, Resolve, Stream},
     primitive::{PdfStream, Primitive},
 };
-
+use pdf::object::Resources;
+use pdf_render::{load_image, BlendMode};
 use crate::{CosFile, PathTrace, Step};
 
 #[macro_export]
@@ -152,19 +153,15 @@ pub fn get_image_by_path(path: &str, file: &CosFile) -> Result<DynamicImage, Str
     let stream = get_pdf_stream_by_path_with_file(path, file)?;
     let resolver = file.resolver();
     let img = t!(ImageXObject::from_stream(stream, &resolver));
-    let data = t!(img.image_data(&resolver));
+    let img_data = t!(load_image(&img, &Resources::default(), &resolver, BlendMode::Overlay));
-    let color_space = img.color_space.as_ref();
+    let h = img_data.height();
-
+    let w = img_data.width();
-    match color_space {
+    let mut data: Vec<u8> = Vec::with_capacity((h * w * 4) as usize);
-        Some(ColorSpace::DeviceRGB) | Some(ColorSpace::CalRGB(_)) => {
+    for x in img_data.rgba_data() {
-            RgbImage::from_raw(img.width, img.height, data.to_vec()).map(DynamicImage::ImageRgb8)
+        data.push(*x);
-        }
-        Some(ColorSpace::DeviceGray) | Some(ColorSpace::CalGray(_)) => {
-            GrayImage::from_raw(img.width, img.height, data.to_vec()).map(DynamicImage::ImageLuma8)
-        }
-        _ => return parse_image_lossy(data),
     }
-    .ok_or_else(|| "Failed to decode image".to_string())
+    let rgba_img = RgbaImage::from_raw(w, h, data).expect("Image conversion failed");
+    Ok(DynamicImage::ImageRgba8(rgba_img))
 }
 
 fn parse_image_lossy(data: Arc<[u8]>) -> Result<DynamicImage, String> {

src-tauri/src/tests.rs

@@ -252,28 +252,5 @@ mod tests {
         );
     }
 
-    use crate::render::{render, render_with_dpi, Renderer};
 
-    #[test]
-    fn test_render() {
-        let file = timed!(
-            FileOptions::cached().open(PDF_SPEC_PATH).unwrap(),
-            "Loading file"
-        );
-        let mut renderer = Renderer::new(&file, 150);
-        let mut rendered_pages: Vec<DynamicImage> = vec![];
-        timed!(
-            for i in 0..1 {
-                let img = timed!(renderer.render(i), format!("render page {}", i));
-                rendered_pages.push(img.unwrap())
-            },
-            "rendering some pages"
-        );
-        timed!(
-            for (num, image) in rendered_pages.iter().enumerate() {
-                image.to_rgb8().save(format!("page_{}.jpg", num)).unwrap()
-            },
-            "saving images"
-        );
-    }
 }

src/components/PageView.svelte

@@ -35,7 +35,7 @@
             </div>
         </Pane>
         <Pane minSize={1}>
-            <RenderedPageView imgB64={state.img_data} {height}></RenderedPageView>
+            <RenderedPageView img={state.img_data} {height}></RenderedPageView>
         </Pane>
     </Splitpanes>
 {:else }

src/components/PrimitiveView.svelte

@@ -2,12 +2,12 @@
     import type FileViewState from "../models/FileViewState.svelte";
     import type Primitive from "../models/Primitive.svelte";
     import PrimitiveIcon from "./PrimitiveIcon.svelte";
-    import StreamEditor from "./StreamEditor.svelte";
+    import {Pane, Splitpanes} from "svelte-splitpanes";
     import StreamDataView from "./StreamDataView.svelte";
 
     const cellH = 29;
     const headerOffset = 24;
-    let { fState, height }: { fState: FileViewState; height: number } =
+    let {fState, height}: { fState: FileViewState; height: number } =
         $props();
     let fillerHeight: number = $state(0);
     let firstEntry = $state(0);
@@ -50,60 +50,72 @@
         fState.selectPath(_path);
     }
 </script>
-
+<Splitpanes  theme="forge-movable">
-{#if prim && prim.children && prim.children.length > 0}
+    <Pane minSize={1}>
-    <div
+        {#if prim && prim.children && prim.children.length > 0}
-        class="overflow-auto"
+            <div
-        onscroll={handleScroll}
+                    class="overflow-auto"
-        style="height: {bodyHeight}px"
+                    onscroll={handleScroll}
-    >
+                    style="height: {bodyHeight}px"
-        <div class="w-[851px]">
+            >
-            <table style="position: relative; top: {scrollY}px">
+                <div class="w-[851px]">
-                <thead>
+                    <table style="position: relative; top: {scrollY}px">
-                    <tr>
+                        <thead>
-                        <td class="page-cell t-header border-forge-prim">Key</td
+                        <tr>
-                        >
+                            <td class="page-cell t-header border-forge-prim">Key
-                        <td class="ref-cell t-header border-forge-prim">Type</td
+                            </td
-                        >
-                        <td class="cell t-header border-forge-sec">Value</td>
-                    </tr>
-                </thead>
-            </table>
-            <div class="container" style="height: {tableHeight}px">
-                <table>
-                    <tbody>
-                        <tr class="filler" style="height: {fillerHeight}px"
-                        ></tr>
-                        {#each entriesToDisplay as entry}
-                            <tr
-                                class:selected={entry.key ===
-                                    locallySelected?.key}
-                                class="row"
-                                onclick={() => handlePrimClick(entry)}
-                                ondblclick={() => handlePrimDbLClick(entry)}
                             >
-                                <td class="page-cell t-data">
+                            <td class="ref-cell t-header border-forge-prim">Type
-                                    <div class="key-field">
+                            </td
-                                        <PrimitiveIcon ptype={entry.ptype} />
+                            >
-                                        <p class="text-left">
+                            <td class="cell t-header border-forge-sec">Value</td>
-                                            {entry.key}
+                        </tr>
-                                        </p>
+                        </thead>
-                                    </div>
+                    </table>
-                                </td>
+                    <div class="container" style="height: {tableHeight}px">
-                                <td class="ref-cell t-data">{entry.ptype}</td>
+                        <table>
-                                <td class="cell t-data">{entry.value}</td>
+                            <tbody>
-                            </tr>
+                            <tr class="filler" style="height: {fillerHeight}px"
-                        {/each}
+                            ></tr>
-                    </tbody>
+                            {#each entriesToDisplay as entry}
-                </table>
+                                <tr
+                                        class:selected={entry.key ===
+                                    locallySelected?.key}
+                                        class="row"
+                                        onclick={() => handlePrimClick(entry)}
+                                        ondblclick={() => handlePrimDbLClick(entry)}
+                                >
+                                    <td class="page-cell t-data">
+                                        <div class="key-field">
+                                            <PrimitiveIcon ptype={entry.ptype}/>
+                                            <p class="text-left">
+                                                {entry.key}
+                                            </p>
+                                        </div>
+                                    </td>
+                                    <td class="ref-cell t-data">{entry.ptype}</td>
+                                    <td class="cell t-data">{entry.value}</td>
+                                </tr>
+                            {/each}
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
             </div>
+        {/if}
+    </Pane>
+    <Pane
+            size={fState.container_prim?.stream_data ? 50 : 0}
+            minSize={fState.container_prim?.stream_data ? 1 : 0}
+            maxSize={fState.container_prim?.stream_data ? 100 : 0}
+    >
+        <div style:height={height} class="bg-forge-dark">
             {#if fState.container_prim?.stream_data}
-                <StreamDataView {fState} {editorHeight}></StreamDataView>
+                <StreamDataView {fState} {height}></StreamDataView>
             {/if}
         </div>
-    </div>
+    </Pane>
-{/if}
+</Splitpanes>
-
 <style lang="postcss">
     .key-field {
         display: flex;

src/components/RenderedPageView.svelte

@@ -1,18 +1,19 @@
 <script lang="ts">
     import ZoomableContainer from "./ZoomableContainer.svelte";
     import { RefreshOutline } from "flowbite-svelte-icons";
+    import {RawImageData} from "../models/RawImageData";
 
-    let { imgB64, height }: { imgB64: string | undefined; height: number } =
+    let { img, height }: { img: RawImageData | undefined; height: number } =
         $props();
 </script>
 
 <div class="page-container" style:height={height + "px"}>
-    {#if !imgB64}
+    {#if !img}
         <div class="loading-container">
             <RefreshOutline class="animate-spin" size="xl" />
         </div>
     {:else}
-        <ZoomableContainer {imgB64} {height} />
+        <ZoomableContainer {img} {height} />
     {/if}
 </div>
 

src/components/StreamDataView.svelte

@@ -5,19 +5,22 @@
     import type {StreamData} from "../models/StreamData.svelte";
     import RenderedPageView from "./RenderedPageView.svelte";
 
-    let {fState, editorHeight}: { fState: FileViewState, editorHeight: number } = $props()
+    let {fState, height}: { fState: FileViewState, height: number } = $props()
-    let streamData: StreamData = $derived(fState.container_prim?.stream_data?);
+    let streamData: StreamData | undefined = $derived(fState.container_prim?.stream_data);
 
 
 </script>
-{#if streamData.type === "Image"}
+{#if streamData}
-    <RenderedPageView imgB64={streamData.data} height={editorHeight}></RenderedPageView>
+    {#if streamData.type === "Image"}
-{:else}
+        <RenderedPageView img={streamData.imageData} {height}></RenderedPageView>
-    <StreamEditor
+    {:else}
-            stream_data={streamData.data}
+        {#if streamData.textData}
-            height={editorHeight}
+            <StreamEditor
-    ></StreamEditor>
+                    stream_data={streamData.textData}
-
+                    {height}
+            ></StreamEditor>
+        {/if}
+    {/if}
 {/if}
 
 <style lang="postcss">

src/components/XRefTable.svelte

@@ -16,7 +16,6 @@
     );
 
     let bodyViewHeight: number = $derived(Math.max(height - headerOffset, 0));
-    $inspect(bodyViewHeight);
     let selectedPath: number | string | undefined = $derived(
         fState.getLastJump(),
     );

src/components/ZoomControls.svelte

@@ -17,7 +17,6 @@
         resetZoom: () => void;
     } = $props();
 
-    $inspect(scale);
 </script>
 
 <div class="controls">

src/components/ZoomableContainer.svelte

@@ -1,11 +1,12 @@
 <script lang="ts">
     import ZoomControls from "../components/ZoomControls.svelte";
+    import { RawImageData } from "../models/RawImageData";
 
     type ZoomableProps = {
         minZoom?: number;
         maxZoom?: number;
         zoomStep?: number;
-        imgB64: string;
+        img: RawImageData;
         height?: string | number;
     };
 
@@ -13,7 +14,7 @@
         minZoom = 0.5,
         maxZoom = 4,
         zoomStep = 0.1,
-        imgB64,
+        img,
         height,
     }: ZoomableProps = $props();
 
@@ -106,6 +107,7 @@
             return () => container.removeEventListener("wheel", handleZoom);
         }
     });
+
 </script>
 
 <div class="relative w-full h-full">
@@ -132,7 +134,7 @@
         >
             <img
                 alt="rendered-page"
-                src={imgB64}
+                src={img.toObjectUrl()}
                 style:max-height={height + "px"}
             />
         </div>

src/models/FileViewState.svelte.ts

@@ -1,13 +1,14 @@
 import type PdfFile from "./PdfFile";
 import type XRefEntry from "./XRefEntry";
 import Primitive from "./Primitive.svelte";
-import { invoke } from "@tauri-apps/api/core";
+import {invoke} from "@tauri-apps/api/core";
 import type XRefTable from "./XRefTable";
-import type { PathSelectedEvent } from "../events/PathSelectedEvent";
+import type {PathSelectedEvent} from "../events/PathSelectedEvent";
-import type { PrimitiveModel } from "./PrimitiveModel";
+import type {PrimitiveModel} from "./PrimitiveModel";
-import { StreamData } from "./StreamData.svelte";
+import {StreamData} from "./StreamData.svelte";
-import { ForgeNotification } from "./ForgeNotification.svelte";
+import {ForgeNotification} from "./ForgeNotification.svelte";
-import { Mutex } from 'async-mutex';
+import {Mutex} from 'async-mutex';
+import {RawImageData} from "./RawImageData";
 
 export default class FileViewState {
     public file: PdfFile;
@@ -72,7 +73,7 @@ export default class FileViewState {
     }
 
     public loadXrefEntries() {
-        invoke<XRefTable>("get_xref_table", { id: this.file.id })
+        invoke<XRefTable>("get_xref_table", {id: this.file.id})
             .then(result => {
                 this.xref_entries = result.entries;
             })
@@ -109,12 +110,12 @@ export default class FileViewState {
             const path = this.formatPaths(this.path) + "/Data";
             container.stream_data = new StreamData(leaf.sub_type)
             if (leaf.sub_type === "Image") {
-                const data = await invoke<string>("get_stream_data_as_image", { id: this.file.id, path: path })
+                const data = await invoke<ArrayBuffer>("get_stream_data_as_image", {id: this.file.id, path: path})
                     .catch(err => this.logError(err))
                 if (!data) return;
-                container.stream_data.setData(data);
+                container.stream_data.setImageData(new RawImageData(data));
             } else {
-                const data = await invoke<string>("get_stream_data_as_string", { id: this.file.id, path: path })
+                const data = await invoke<string>("get_stream_data_as_string", {id: this.file.id, path: path})
                     .catch(err => this.logError(err))
                 if (!data) return;
                 container.stream_data.setData(data);

src/models/PageViewState.svelte.ts

@@ -1,12 +1,13 @@
 import type ContentModel from "./ContentModel.svelte";
 import {invoke} from "@tauri-apps/api/core";
+import {RawImageData} from "./RawImageData";
 
 export class PageViewState {
     file_id: string;
     page_num: number;
     errorHandler: (arg0: string) => void;
 
-    img_data: string | undefined = $state();
+    img_data: RawImageData | undefined = $state();
     contents: string = $state("");
 
     constructor(file_id: string, page_num: number, errorHandler: (arg0: string) => void) {
@@ -26,12 +27,12 @@ export class PageViewState {
     }
 
     public async loadImage() {
-        let result = await invoke<string>("get_page_by_num", {
+        let result = await invoke<ArrayBuffer>("get_page_by_num", {
             id: this.file_id,
             num: this.page_num,
         }).catch(err => this.errorHandler(err));
         if (result) {
-            this.img_data = result;
+            this.img_data = new RawImageData(result);
         }
     }
 

src/models/RawImageData.ts

@@ -0,0 +1,51 @@
+export class RawImageData {
+    height: number;
+    width: number;
+    pixels: ArrayBuffer;
+    private _objectUrl: string | null = null;
+
+    constructor(buff: ArrayBuffer) {
+        const view = new DataView(buff);
+        this.width = view.getUint32(0, true); // true for little-endian
+        this.height = view.getUint32(4, true);
+        this.pixels = buff.slice(8);
+    }
+
+    private createImageData(): ImageData {
+        const uint8Array = new Uint8Array(this.pixels);
+        return new ImageData(
+            new Uint8ClampedArray(uint8Array),
+            this.width,
+            this.height
+            ,
+        );
+    }
+
+    toObjectUrl(): string {
+        if (this._objectUrl) {
+            return this._objectUrl;
+        }
+
+        const canvas = document.createElement('canvas');
+        canvas.width = this.width;
+        canvas.height = this.height;
+
+        const ctx = canvas.getContext('2d');
+        if (!ctx) {
+            throw new Error('Could not get canvas context');
+        }
+
+        const imageData = this.createImageData();
+        ctx.putImageData(imageData, 0, 0);
+
+        this._objectUrl = canvas.toDataURL('image/png');
+        return this._objectUrl;
+    }
+
+    dispose() {
+        if (this._objectUrl) {
+            URL.revokeObjectURL(this._objectUrl);
+            this._objectUrl = null;
+        }
+    }
+}

src/models/StreamData.svelte.ts

@@ -1,12 +1,20 @@
+import type {RawImageData} from "./RawImageData";
+
 export class StreamData {
     // Image, Text
     public type: string;
-    public data: string = $state("");
+    public textData: string | undefined = $state();
+    public imageData: RawImageData | undefined = $state();
+
     constructor(type: string) {
         this.type = type;
     }
 
     public setData(data: string) {
-        this.data = data;
+        this.textData = data;
+    }
+
+    public setImageData(data: RawImageData) {
+        this.imageData = data;
     }
 }