Merge pull request #21474 from calixteman/rich_media

Add support for RichMedia annotations

src/core/annotation.js

@@ -287,6 +287,9 @@ class AnnotationFactory {
       case "FileAttachment":
         return new FileAttachmentAnnotation(parameters);
 
+      case "RichMedia":
+        return new RichMediaAnnotation(parameters);
+
       default:
         if (!collectFields) {
           if (!subtype) {
@@ -5454,6 +5457,162 @@ class FileAttachmentAnnotation extends MarkupAnnotation {
   }
 }
 
+class RichMediaAnnotation extends Annotation {
+  constructor(params) {
+    super(params);
+
+    this.data.noHTML = true;
+
+    const { dict, xref, annotationGlobals } = params;
+
+    const content = dict.get("RichMediaContent");
+    if (!(content instanceof Dict)) {
+      return;
+    }
+
+    const asset = RichMediaAnnotation.#findAsset(content, xref);
+    if (!asset) {
+      warn("RichMedia annotation has no playable asset.");
+      return;
+    }
+    const { assetRef, assetDict, filename, contentType } = asset;
+
+    let contentRef = assetRef;
+    if (!(contentRef instanceof Ref)) {
+      contentRef = FileSpec.pickPlatformItem(
+        assetDict.get("EF"),
+        /* raw = */ true
+      );
+    }
+    const fileId =
+      contentRef instanceof Ref
+        ? annotationGlobals.pdfManager.pdfDocument.catalog?.getAttachmentIdForAnnotation(
+            contentRef
+          )
+        : undefined;
+
+    this.data.noHTML = false;
+    this.data.richMedia = { fileId, filename, contentType };
+  }
+
+  /**
+   * Locate the primary playable embedded media asset.
+   *
+   * Per the spec (ISO 32000-2, 13.7), the asset to play is selected through
+   * `Configurations -> Instances -> Asset`. We pick the first instance with a
+   * natively playable asset rather than honoring the default configuration
+   * indicated by `RichMediaSettings`/activation; this keeps selection simple
+   * and matches the common single-configuration case. The `/Assets` name tree
+   * merely enumerates every embedded file; we don't use it as a fallback, since
+   * Acrobat itself won't play media that's only reachable that way. Flash
+   * instances are skipped, since they can't be played natively.
+   *
+   * @returns {{
+   *   assetRef: Ref | null,
+   *   assetDict: Dict,
+   *   filename: string,
+   *   contentType: string,
+   * } | null}
+   */
+  static #findAsset(content, xref) {
+    const configurations = content.get("Configurations");
+    if (!Array.isArray(configurations)) {
+      return null;
+    }
+    for (const configRef of configurations) {
+      const config = xref.fetchIfRef(configRef);
+      if (!(config instanceof Dict)) {
+        continue;
+      }
+      const instances = config.get("Instances");
+      if (!Array.isArray(instances)) {
+        continue;
+      }
+      for (const instanceRef of instances) {
+        const instance = xref.fetchIfRef(instanceRef);
+        if (!(instance instanceof Dict)) {
+          continue;
+        }
+        // Skip Flash instances: it's obsolete.
+        if (isName(instance.get("Subtype"), "Flash")) {
+          // Flash has been supported (see PDF 1.7 Extension Level 3).
+          continue;
+        }
+        const rawAsset = instance.getRaw("Asset");
+        const asset = xref.fetchIfRef(rawAsset);
+        if (!(asset instanceof Dict)) {
+          continue;
+        }
+        const { filename } = new FileSpec(asset).serializable;
+        const contentType = RichMediaAnnotation.#getContentType(
+          asset,
+          filename
+        );
+        if (!contentType) {
+          continue;
+        }
+        return {
+          assetRef: rawAsset instanceof Ref ? rawAsset : null,
+          assetDict: asset,
+          filename,
+          contentType,
+        };
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Determine the MIME type used to build the `<video>`/`<audio>` element.
+   *
+   * Per the spec (ISO 32000-2, 7.11.4) an embedded file stream declares its
+   * MIME type through its own `/Subtype`, with characters not allowed in a
+   * name hex-escaped (e.g. `/video#2Fmp4` -> `video/mp4`). We trust that when
+   * it names an audio/video type, and otherwise fall back to mapping the
+   * filename extension. Returns `null` when the asset isn't a recognized
+   * audio/video type (e.g. Flash `.swf` or 3D models), so we don't build a
+   * player that can't play anything.
+   *
+   * @param {Dict} assetDict
+   * @param {string} filename
+   * @returns {string | null}
+   */
+  static #getContentType(assetDict, filename) {
+    // The embedded file stream is keyed like a file-spec platform item.
+    const stream = FileSpec.pickPlatformItem(assetDict.get("EF"));
+    const subtype =
+      stream instanceof BaseStream ? stream.dict?.get("Subtype") : null;
+    if (subtype instanceof Name && /^(?:video|audio)\//.test(subtype.name)) {
+      return subtype.name;
+    }
+
+    const ext = filename.split(".").at(-1)?.toLowerCase();
+    switch (ext) {
+      case "mp4":
+      case "m4v":
+        return "video/mp4";
+      case "webm":
+        return "video/webm";
+      case "ogv":
+        return "video/ogg";
+      case "mov":
+        return "video/quicktime";
+      case "mp3":
+        return "audio/mpeg";
+      case "m4a":
+        return "audio/mp4";
+      case "wav":
+        return "audio/wav";
+      case "oga":
+      case "ogg":
+        return "audio/ogg";
+      default:
+        return null;
+    }
+  }
+}
+
 export {
   Annotation,
   AnnotationBorderStyle,

src/display/annotation_layer.js

@@ -158,6 +158,9 @@ class AnnotationElementFactory {
       case AnnotationType.FILEATTACHMENT:
         return new FileAttachmentAnnotationElement(parameters);
 
+      case AnnotationType.RICHMEDIA:
+        return new RichMediaAnnotationElement(parameters);
+
       default:
         return new AnnotationElement(parameters);
     }
@@ -399,7 +402,8 @@ class AnnotationElement {
     container.setAttribute("data-annotation-id", data.id);
     if (
       !(this instanceof WidgetAnnotationElement) &&
-      !(this instanceof LinkAnnotationElement)
+      !(this instanceof LinkAnnotationElement) &&
+      !(this instanceof RichMediaAnnotationElement)
     ) {
       container.tabIndex = 0;
     }
@@ -908,6 +912,12 @@ class AnnotationElement {
   get height() {
     return this.data.rect[3] - this.data.rect[1];
   }
+
+  _setBackgroundColor(element) {
+    const color = this.data.backgroundColor || null;
+    element.style.backgroundColor =
+      color === null ? "transparent" : Util.makeHexColor(...color);
+  }
 }
 
 class EditorAnnotationElement extends AnnotationElement {
@@ -1391,12 +1401,6 @@ class WidgetAnnotationElement extends AnnotationElement {
     }
   }
 
-  _setBackgroundColor(element) {
-    const color = this.data.backgroundColor || null;
-    element.style.backgroundColor =
-      color === null ? "transparent" : Util.makeHexColor(...color);
-  }
-
   /**
    * Apply text styles to the text in the element.
    *
@@ -3796,6 +3800,138 @@ class FileAttachmentAnnotationElement extends AnnotationElement {
   }
 }
 
+class RichMediaAnnotationElement extends AnnotationElement {
+  #abortController = new AbortController();
+
+  #contentUrl = null;
+
+  #media = null;
+
+  constructor(parameters) {
+    super(parameters, { isRenderable: !!parameters.data.richMedia });
+  }
+
+  render() {
+    this.container.classList.add("richMediaAnnotation");
+
+    const { filename } = this.data.richMedia;
+
+    // The annotation's appearance (a poster image) is painted on the canvas;
+    // overlay a play button that loads the embedded media on demand.
+    const button = document.createElement("button");
+    button.className = "richMediaPlayButton";
+    button.type = "button";
+    button.title = button.ariaLabel = filename;
+    button.addEventListener("click", () => this.#load(button), {
+      signal: this.#abortController.signal,
+    });
+
+    this.container.append(button);
+    return this.container;
+  }
+
+  async #load(button) {
+    const { fileId, filename, contentType } = this.data.richMedia;
+    button.disabled = true;
+
+    let content;
+    try {
+      content = await this.linkService.getAttachmentContent(fileId);
+    } catch {
+      // Leave the play button in place so the load can be retried.
+      return;
+    } finally {
+      button.disabled = false;
+    }
+    if (!content || !button.isConnected) {
+      return;
+    }
+
+    const { signal } = this.#abortController;
+    const url = URL.createObjectURL(new Blob([content], { type: contentType }));
+    this.#contentUrl = url;
+    const isAudio = contentType.startsWith("audio/");
+    const media = document.createElement(isAudio ? "audio" : "video");
+    this.#media = media;
+    media.className = "richMediaContent";
+    this._setBackgroundColor(media);
+    media.src = url;
+    media.title = filename;
+    media.controls = true;
+    media.autoplay = true;
+    media.tabIndex = 0;
+    if (isAudio) {
+      // An `<audio>` element's controls would otherwise always be visible;
+      // only show them while the section is hovered or focused.
+      let hovered = false;
+      let focused = false;
+      const updateControls = () => {
+        media.controls = hovered || focused;
+      };
+      this.container.addEventListener(
+        "pointerenter",
+        () => {
+          hovered = true;
+          updateControls();
+        },
+        { signal }
+      );
+      this.container.addEventListener(
+        "pointerleave",
+        () => {
+          hovered = false;
+          updateControls();
+        },
+        { signal }
+      );
+      this.container.addEventListener(
+        "focusin",
+        () => {
+          focused = true;
+          updateControls();
+        },
+        { signal }
+      );
+      this.container.addEventListener(
+        "focusout",
+        () => {
+          focused = false;
+          updateControls();
+        },
+        { signal }
+      );
+    }
+    // Release the object URL once the browser no longer needs the source.
+    media.addEventListener("emptied", () => this.#revokeContentUrl(url), {
+      once: true,
+      signal,
+    });
+
+    button.replaceWith(media);
+    media.play().catch(() => {});
+  }
+
+  #revokeContentUrl(url = this.#contentUrl) {
+    if (url && url === this.#contentUrl) {
+      URL.revokeObjectURL(url);
+      this.#contentUrl = null;
+    }
+  }
+
+  destroy() {
+    // Aborting also removes the `emptied` listener below, so revoke the object
+    // URL explicitly rather than relying on the teardown triggering it.
+    this.#abortController.abort();
+    if (this.#media) {
+      this.#media.pause();
+      this.#media.removeAttribute("src");
+      this.#media.load();
+      this.#media = null;
+    }
+    this.#revokeContentUrl();
+  }
+}
+
 /**
  * @typedef {Object} AnnotationLayerParameters
  * @property {PageViewport} viewport
@@ -4126,6 +4262,18 @@ class AnnotationLayer {
     layer.hidden = false;
   }
 
+  destroy() {
+    for (const element of this.#elements) {
+      element.destroy?.();
+      this.#accessibilityManager?.removePointerInTextLayer(
+        element.contentElement
+      );
+    }
+    this.#elements.length = 0;
+    this.#editableAnnotations.clear();
+    this.div.replaceChildren();
+  }
+
   #setAnnotationCanvasMap() {
     if (!this.#annotationCanvasMap) {
       return;

src/shared/util.js

@@ -167,6 +167,7 @@ const AnnotationType = {
   WATERMARK: 24,
   THREED: 25,
   REDACT: 26,
+  RICHMEDIA: 27,
 };
 
 const AnnotationReplyType = {

test/integration/annotation_spec.mjs

@@ -967,3 +967,62 @@ a dynamic compiler for JavaScript based on our`);
     });
   });
 });
+
+describe("RichMedia annotation", () => {
+  describe("multimedia_annotations.pdf", () => {
+    let pages;
+
+    beforeEach(async () => {
+      pages = await loadAndWait(
+        "multimedia_annotations.pdf",
+        getAnnotationSelector("4R")
+      );
+    });
+
+    afterEach(async () => {
+      await closePages(pages);
+    });
+
+    it("must play the embedded video when clicking the play button", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const annotationSelector = getAnnotationSelector("4R");
+          const buttonSelector = `${annotationSelector} .richMediaPlayButton`;
+          const videoSelector = `${annotationSelector} video.richMediaContent`;
+
+          // Initially only the play button (over the poster) is shown.
+          await page.waitForSelector(buttonSelector, { timeout: 0 });
+          await page.click(buttonSelector);
+
+          // Clicking it loads the embedded media into a <video> element.
+          await page.waitForSelector(videoSelector, { timeout: 0 });
+          const hasSource = await page.$eval(videoSelector, el =>
+            el.src.startsWith("blob:")
+          );
+          expect(hasSource).withContext(`In ${browserName}`).toEqual(true);
+        })
+      );
+    });
+
+    it("must play the embedded audio when clicking the play button", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const annotationSelector = getAnnotationSelector("5R");
+          const buttonSelector = `${annotationSelector} .richMediaPlayButton`;
+          const audioSelector = `${annotationSelector} audio.richMediaContent`;
+
+          // Initially only the play button is shown.
+          await page.waitForSelector(buttonSelector, { timeout: 0 });
+          await page.click(buttonSelector);
+
+          // Clicking it loads the embedded media into an <audio> element.
+          await page.waitForSelector(audioSelector, { timeout: 0 });
+          const hasSource = await page.$eval(audioSelector, el =>
+            el.src.startsWith("blob:")
+          );
+          expect(hasSource).withContext(`In ${browserName}`).toEqual(true);
+        })
+      );
+    });
+  });
+});

test/integration/presentation_mode_spec.mjs

@@ -17,6 +17,7 @@ import {
   awaitPromise,
   closePages,
   createPromise,
+  getAnnotationSelector,
   loadAndWait,
   waitForTimeout,
 } from "./test_utils.mjs";
@@ -193,4 +194,98 @@ describe("PDFPresentationMode", () => {
       );
     });
   });
+
+  describe("RichMedia annotations", () => {
+    let pages;
+
+    beforeEach(async () => {
+      pages = await loadAndWait(
+        "multimedia_annotations.pdf",
+        getAnnotationSelector("4R"),
+        100
+      );
+    });
+
+    afterEach(async () => {
+      await closePages(pages);
+    });
+
+    it("must handle clicking embedded media in presentation mode", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const annotationSelector = getAnnotationSelector("4R");
+          const videoSelector = `${annotationSelector} video.richMediaContent`;
+
+          await page.evaluate(() => {
+            window.__richMediaNavigationCalls = 0;
+            const { pdfViewer } = window.PDFViewerApplication;
+            for (const name of ["nextPage", "previousPage"]) {
+              const original = pdfViewer[name].bind(pdfViewer);
+              pdfViewer[name] = (...args) => {
+                window.__richMediaNavigationCalls++;
+                return original(...args);
+              };
+            }
+          });
+
+          await enterPresentationMode(page);
+          await page.waitForSelector(
+            `${annotationSelector} .richMediaPlayButton`,
+            { visible: true }
+          );
+          await page.click(`${annotationSelector} .richMediaPlayButton`);
+          await page.waitForSelector(videoSelector, { timeout: 0 });
+
+          const result = await page.$eval(videoSelector, el => ({
+            hasSource: el.src.startsWith("blob:"),
+          }));
+          expect(result)
+            .withContext(`In ${browserName}`)
+            .toEqual({ hasSource: true });
+
+          const interactionResult = await page.$eval(videoSelector, el => {
+            const createTouchEvent = (type, x, y, hasTouch = true) => {
+              const evt = new Event(type, {
+                bubbles: true,
+                cancelable: true,
+              });
+              Object.defineProperty(evt, "touches", {
+                value: hasTouch ? [{ pageX: x, pageY: y }] : [],
+              });
+              return evt;
+            };
+
+            const wheelEvent = new WheelEvent("wheel", {
+              bubbles: true,
+              cancelable: true,
+              deltaY: 100,
+            });
+            el.dispatchEvent(wheelEvent);
+
+            const touchMoveEvent = createTouchEvent("touchmove", 20, 200);
+            el.dispatchEvent(createTouchEvent("touchstart", 200, 200));
+            el.dispatchEvent(touchMoveEvent);
+            el.dispatchEvent(
+              createTouchEvent("touchend", 20, 200, /* hasTouch = */ false)
+            );
+
+            return {
+              navigationCalls: window.__richMediaNavigationCalls,
+              pageNumber: window.PDFViewerApplication.page,
+              touchMoveDefaultPrevented: touchMoveEvent.defaultPrevented,
+              wheelDefaultPrevented: wheelEvent.defaultPrevented,
+            };
+          });
+          expect(interactionResult).withContext(`In ${browserName}`).toEqual({
+            navigationCalls: 0,
+            pageNumber: 1,
+            touchMoveDefaultPrevented: false,
+            wheelDefaultPrevented: false,
+          });
+
+          await exitPresentationMode(page, browserName);
+        })
+      );
+    });
+  });
 });

test/pdfs/.gitignore

@@ -938,3 +938,4 @@
 !opt_demo.pdf
 !bug1873345.pdf
 !cff_bluescale_small_zones.pdf
+!multimedia_annotations.pdf

test/unit/annotation_spec.js

@@ -4596,6 +4596,317 @@ describe("annotation", function () {
     });
   });
 
+  describe("RichMediaAnnotation", function () {
+    function createAssetDict(filename, mimeSubtype = null) {
+      let streamDict = null;
+      if (mimeSubtype) {
+        streamDict = new Dict();
+        streamDict.set("Type", Name.get("EmbeddedFile"));
+        streamDict.set("Subtype", Name.get(mimeSubtype));
+      }
+      const embeddedFileDict = new Dict();
+      embeddedFileDict.set("F", new StringStream("", streamDict));
+
+      const fileSpecDict = new Dict();
+      fileSpecDict.set("Type", Name.get("Filespec"));
+      fileSpecDict.set("EF", embeddedFileDict);
+      fileSpecDict.set("UF", filename);
+      return fileSpecDict;
+    }
+
+    function createAnnotation(contentDict, refNum) {
+      const dict = new Dict();
+      dict.set("Type", Name.get("Annot"));
+      dict.set("Subtype", Name.get("RichMedia"));
+      dict.set("RichMediaContent", contentDict);
+      return { ref: Ref.get(refNum, 0), data: dict };
+    }
+
+    it("should parse the media asset from Configurations -> Instances", async function () {
+      const assetRef = Ref.get(100, 0);
+      const assetDict = createAssetDict("demo.mp4");
+
+      const instanceDict = new Dict();
+      instanceDict.set("Type", Name.get("RichMediaInstance"));
+      instanceDict.set("Subtype", Name.get("Video"));
+      instanceDict.set("Asset", assetRef);
+
+      const configDict = new Dict();
+      configDict.set("Type", Name.get("RichMediaConfiguration"));
+      configDict.set("Subtype", Name.get("Video"));
+      configDict.set("Instances", [instanceDict]);
+
+      const contentDict = new Dict();
+      contentDict.set("Type", Name.get("RichMediaContent"));
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 101);
+      const xref = new XRefMock([
+        { ref: assetRef, data: assetDict },
+        annotation,
+      ]);
+      assetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.annotationType).toEqual(AnnotationType.RICHMEDIA);
+      expect(data.noHTML).toEqual(false);
+      expect(data.richMedia).toEqual({
+        fileId: "attachmentRef:100R",
+        filename: "demo.mp4",
+        contentType: "video/mp4",
+      });
+    });
+
+    it("should derive the content type from the asset extension", async function () {
+      const assetRef = Ref.get(110, 0);
+      const assetDict = createAssetDict("clip.mp3");
+
+      const instanceDict = new Dict();
+      instanceDict.set("Subtype", Name.get("Sound"));
+      instanceDict.set("Asset", assetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [instanceDict]);
+
+      const contentDict = new Dict();
+      contentDict.set("Type", Name.get("RichMediaContent"));
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 111);
+      const xref = new XRefMock([
+        { ref: assetRef, data: assetDict },
+        annotation,
+      ]);
+      assetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.richMedia).toEqual({
+        fileId: "attachmentRef:110R",
+        filename: "clip.mp3",
+        contentType: "audio/mpeg",
+      });
+    });
+
+    it("should prefer the MIME type declared on the embedded file stream", async function () {
+      const assetRef = Ref.get(150, 0);
+      // The extension is unhelpful, but the stream declares `video/webm`
+      // (in a real file as `/Subtype /video#2Fwebm`).
+      const assetDict = createAssetDict("movie.bin", "video/webm");
+
+      const instanceDict = new Dict();
+      instanceDict.set("Subtype", Name.get("Video"));
+      instanceDict.set("Asset", assetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [instanceDict]);
+
+      const contentDict = new Dict();
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 151);
+      const xref = new XRefMock([
+        { ref: assetRef, data: assetDict },
+        annotation,
+      ]);
+      assetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.richMedia.filename).toEqual("movie.bin");
+      expect(data.richMedia.contentType).toEqual("video/webm");
+    });
+
+    it("should skip a Flash instance and prefer a playable one", async function () {
+      const flashAssetRef = Ref.get(120, 0);
+      const flashInstance = new Dict();
+      flashInstance.set("Subtype", Name.get("Flash"));
+      flashInstance.set("Asset", flashAssetRef);
+
+      const videoAssetRef = Ref.get(121, 0);
+      const videoAssetDict = createAssetDict("demo.mp4");
+      const videoInstance = new Dict();
+      videoInstance.set("Subtype", Name.get("Video"));
+      videoInstance.set("Asset", videoAssetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [flashInstance, videoInstance]);
+
+      const contentDict = new Dict();
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 122);
+      const xref = new XRefMock([
+        { ref: flashAssetRef, data: createAssetDict("movie.swf") },
+        { ref: videoAssetRef, data: videoAssetDict },
+        annotation,
+      ]);
+      videoAssetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.richMedia.filename).toEqual("demo.mp4");
+      expect(data.richMedia.contentType).toEqual("video/mp4");
+    });
+
+    it("should skip an unsupported instance and prefer a playable one", async function () {
+      const modelAssetRef = Ref.get(170, 0);
+      const modelInstance = new Dict();
+      modelInstance.set("Asset", modelAssetRef);
+
+      const videoAssetRef = Ref.get(171, 0);
+      const videoAssetDict = createAssetDict("demo.mp4");
+      const videoInstance = new Dict();
+      videoInstance.set("Subtype", Name.get("Video"));
+      videoInstance.set("Asset", videoAssetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [modelInstance, videoInstance]);
+
+      const contentDict = new Dict();
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 172);
+      const xref = new XRefMock([
+        { ref: modelAssetRef, data: createAssetDict("model.u3d") },
+        { ref: videoAssetRef, data: videoAssetDict },
+        annotation,
+      ]);
+      videoAssetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.richMedia.filename).toEqual("demo.mp4");
+      expect(data.richMedia.contentType).toEqual("video/mp4");
+    });
+
+    it("should not create media data for a Flash (.swf) asset", async function () {
+      const assetRef = Ref.get(130, 0);
+      const assetDict = createAssetDict("movie.swf");
+
+      // The instance isn't flagged `/Flash`, so it isn't skipped during
+      // selection; the `.swf` extension is what rejects it.
+      const instanceDict = new Dict();
+      instanceDict.set("Asset", assetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [instanceDict]);
+
+      const contentDict = new Dict();
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 131);
+      const xref = new XRefMock([
+        { ref: assetRef, data: assetDict },
+        annotation,
+      ]);
+      assetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.annotationType).toEqual(AnnotationType.RICHMEDIA);
+      expect(data.noHTML).toEqual(true);
+      expect(data.richMedia).toBeUndefined();
+    });
+
+    it("should reuse the media file id when parsed more than once", async function () {
+      const assetRef = Ref.get(160, 0);
+      const assetDict = createAssetDict("demo.mp4");
+
+      const instanceDict = new Dict();
+      instanceDict.set("Subtype", Name.get("Video"));
+      instanceDict.set("Asset", assetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [instanceDict]);
+
+      const contentDict = new Dict();
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 161);
+      const xref = new XRefMock([
+        { ref: assetRef, data: assetDict },
+        annotation,
+      ]);
+      assetDict.assignXref(xref);
+
+      const first = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      const second = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+
+      // The id is derived from the asset reference, so parsing twice reuses it
+      // rather than registering the asset under a second id.
+      expect(first.data.richMedia.fileId).toEqual("attachmentRef:160R");
+      expect(second.data.richMedia.fileId).toEqual(first.data.richMedia.fileId);
+    });
+
+    it("should not create media data for an unsupported asset type", async function () {
+      const assetRef = Ref.get(140, 0);
+      // A 3D model isn't playable through a `<video>`/`<audio>` element.
+      const assetDict = createAssetDict("model.u3d");
+
+      const instanceDict = new Dict();
+      instanceDict.set("Asset", assetRef);
+
+      const configDict = new Dict();
+      configDict.set("Instances", [instanceDict]);
+
+      const contentDict = new Dict();
+      contentDict.set("Configurations", [configDict]);
+
+      const annotation = createAnnotation(contentDict, 141);
+      const xref = new XRefMock([
+        { ref: assetRef, data: assetDict },
+        annotation,
+      ]);
+      assetDict.assignXref(xref);
+
+      const { data } = await AnnotationFactory.create(
+        xref,
+        annotation.ref,
+        annotationGlobalsMock,
+        idFactoryMock
+      );
+      expect(data.annotationType).toEqual(AnnotationType.RICHMEDIA);
+      expect(data.noHTML).toEqual(true);
+      expect(data.richMedia).toBeUndefined();
+    });
+  });
+
   describe("PopupAnnotation", function () {
     it("should inherit properties from its parent", async function () {
       const parentDict = new Dict();

web/annotation_layer_builder.css

@@ -328,6 +328,47 @@
     width: 100%;
   }
 
+  .richMediaAnnotation {
+    pointer-events: auto;
+
+    .richMediaContent {
+      width: 100%;
+    }
+
+    video.richMediaContent {
+      height: 100%;
+      object-fit: contain;
+    }
+
+    audio.richMediaContent {
+      /* An `<audio>` element has a fixed-height control bar; pin it to the
+         bottom of the section so it lines up with where `<video>` shows its
+         own bar. */
+      position: absolute;
+      bottom: 0;
+    }
+
+    .richMediaPlayButton {
+      height: 100%;
+      width: 100%;
+      margin: 0;
+      padding: 0;
+      border: none;
+      cursor: pointer;
+      background-color: transparent;
+
+      /* A centered, semi-transparent play triangle. */
+      background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Ccircle cx='50' cy='50' r='40' fill='rgba(0,0,0,0.55)'/%3E%3Cpath d='M40 32 L70 50 L40 68 Z' fill='white'/%3E%3C/svg%3E");
+      background-position: center;
+      background-repeat: no-repeat;
+      background-size: calc(48px * var(--total-scale-factor));
+
+      &:hover {
+        background-color: rgba(255 255 255 / 0.1);
+      }
+    }
+  }
+
   .popupAnnotation {
     position: absolute;
     font-size: calc(9px * var(--total-scale-factor));

web/annotation_layer_builder.js

@@ -222,6 +222,7 @@ class AnnotationLayerBuilder {
 
     this.#eventAC?.abort();
     this.#eventAC = null;
+    this.annotationLayer?.destroy();
   }
 
   refreshCanvases() {
@@ -282,7 +283,10 @@ class AnnotationLayerBuilder {
         return;
     }
     for (const section of this.div.childNodes) {
-      if (section.hasAttribute("data-internal-link")) {
+      if (
+        section.hasAttribute("data-internal-link") ||
+        section.classList.contains("richMediaAnnotation")
+      ) {
         continue;
       }
       section.inert = disableFormElements;

web/pdf_presentation_mode.js

@@ -126,6 +126,9 @@ class PDFPresentationMode {
     if (!this.active) {
       return;
     }
+    if (evt.target.closest?.(".richMediaAnnotation")) {
+      return;
+    }
     evt.preventDefault();
 
     const delta = normalizeWheelEventDelta(evt);
@@ -246,6 +249,10 @@ class PDFPresentationMode {
     ) {
       return;
     }
+    // Allow interacting with embedded media controls rather than advancing.
+    if (evt.target.closest?.(".richMediaAnnotation")) {
+      return;
+    }
     // Unless an internal link was clicked, advance one page.
     evt.preventDefault();
 
@@ -293,6 +300,10 @@ class PDFPresentationMode {
     if (!this.active) {
       return;
     }
+    if (evt.target.closest?.(".richMediaAnnotation")) {
+      this.touchSwipeState = null;
+      return;
+    }
     if (evt.touches.length > 1) {
       // Multiple touch points detected; cancel the swipe.
       this.touchSwipeState = null;

web/viewer-geckoview.html

@@ -29,7 +29,7 @@ See https://github.com/adobe-type-tools/cmap-resources
     <!--#if MOZCENTRAL-->
     <!--<meta
       http-equiv="Content-Security-Policy"
-      content="default-src 'none'; script-src resource: 'wasm-unsafe-eval'; worker-src resource:; style-src resource:; img-src resource: blob: data:; font-src resource:; connect-src resource:; base-uri 'none'; form-action 'none';"
+      content="default-src 'none'; script-src resource: 'wasm-unsafe-eval'; worker-src resource:; style-src resource:; img-src resource: blob: data:; media-src blob:; font-src resource:; connect-src resource:; base-uri 'none'; form-action 'none';"
     />-->
     <!--#endif-->
 

web/viewer.css

@@ -236,7 +236,8 @@ body {
   user-select: none;
 }
 
-.pdfPresentationMode:fullscreen section:not([data-internal-link]) {
+.pdfPresentationMode:fullscreen
+  section:not([data-internal-link], .richMediaAnnotation) {
   pointer-events: none;
 }
 

web/viewer.html

@@ -33,22 +33,22 @@ See https://github.com/adobe-type-tools/cmap-resources
     <!--<link rel="icon" type="image/svg+xml" href="chrome://global/skin/icons/pdf.svg" />-->
     <!--<meta
       http-equiv="Content-Security-Policy"
-      content="default-src 'none'; script-src resource: 'wasm-unsafe-eval'; worker-src resource:; style-src resource:; img-src resource: blob: data:; font-src resource:; connect-src resource:; base-uri 'none'; form-action 'none';"
+      content="default-src 'none'; script-src resource: 'wasm-unsafe-eval'; worker-src resource:; style-src resource:; img-src resource: blob: data:; media-src blob:; font-src resource:; connect-src resource:; base-uri 'none'; form-action 'none';"
     />-->
     <!--#elif TESTING-->
     <!--<meta
       http-equiv="Content-Security-Policy"
-      content="default-src 'none'; script-src 'self' 'wasm-unsafe-eval' 'unsafe-eval'; worker-src 'self' blob:; style-src 'self'; img-src 'self' blob: data:; font-src 'self' data:; connect-src * blob: data:; base-uri 'self'; form-action 'none';"
+      content="default-src 'none'; script-src 'self' 'wasm-unsafe-eval' 'unsafe-eval'; worker-src 'self' blob:; style-src 'self'; img-src 'self' blob: data:; media-src blob:; font-src 'self' data:; connect-src * blob: data:; base-uri 'self'; form-action 'none';"
     />-->
     <!--#elif CHROME-->
     <!--<meta
       http-equiv="Content-Security-Policy"
-      content="default-src 'none'; script-src 'self' 'wasm-unsafe-eval'; worker-src 'self' blob:; style-src 'self'; img-src 'self' blob: data:; font-src 'self' data:; connect-src * file: chrome-extension: blob: data: filesystem: drive:; base-uri 'self'; form-action 'none';"
+      content="default-src 'none'; script-src 'self' 'wasm-unsafe-eval'; worker-src 'self' blob:; style-src 'self'; img-src 'self' blob: data:; media-src blob:; font-src 'self' data:; connect-src * file: chrome-extension: blob: data: filesystem: drive:; base-uri 'self'; form-action 'none';"
     />-->
     <!--#else-->
     <!--<meta
       http-equiv="Content-Security-Policy"
-      content="default-src 'none'; script-src 'self' 'wasm-unsafe-eval'; worker-src 'self' blob:; style-src 'self'; img-src 'self' blob: data:; font-src 'self' data:; connect-src * blob: data:; base-uri 'none'; form-action 'none';"
+      content="default-src 'none'; script-src 'self' 'wasm-unsafe-eval'; worker-src 'self' blob:; style-src 'self'; img-src 'self' blob: data:; media-src blob:; font-src 'self' data:; connect-src * blob: data:; base-uri 'none'; form-action 'none';"
     />-->
     <!--#endif-->