diff --git a/src/core/annotation.js b/src/core/annotation.js
index ecee49c3d..232c21ec5 100644
--- a/src/core/annotation.js
+++ b/src/core/annotation.js
@@ -1335,6 +1335,8 @@ class Annotation {
 
     const text = [];
     const buffer = [];
+    let firstPositionX = Infinity;
+    let firstPositionY = Infinity;
     let firstPosition = null;
     const sink = {
       desiredSize: Math.Infinity,
@@ -1345,7 +1347,8 @@ class Annotation {
           if (item.str === undefined) {
             continue;
           }
-          firstPosition ||= item.transform.slice(-2);
+          firstPositionX = Math.min(firstPositionX, item.transform[4]);
+          firstPositionY = Math.min(firstPositionY, item.transform[5]);
           buffer.push(item.str);
           if (item.hasEOL) {
             text.push(buffer.join("").trimEnd());
@@ -1366,6 +1369,10 @@ class Annotation {
     });
     this.reset();
 
+    if (firstPositionX !== Infinity) {
+      firstPosition = [firstPositionX, firstPositionY];
+    }
+
     if (buffer.length) {
       text.push(buffer.join("").trimEnd());
     }
diff --git a/src/core/default_appearance.js b/src/core/default_appearance.js
index 0c1d0f746..6b36572d6 100644
--- a/src/core/default_appearance.js
+++ b/src/core/default_appearance.js
@@ -25,6 +25,7 @@ import {
   LINE_FACTOR,
   OPS,
   shadow,
+  Util,
   warn,
 } from "../shared/util.js";
 import { ColorSpaceUtils } from "./colorspace_utils.js";
@@ -144,7 +145,8 @@ class AppearanceStreamEvaluator extends EvaluatorPreprocessor {
             result = stack.pop() || result;
             break;
           case OPS.setTextMatrix:
-            result.scaleFactor *= Math.hypot(args[0], args[1]);
+            const tm = Util.transform(this.stateManager.state.ctm, args);
+            result.scaleFactor *= Math.hypot(tm[0], tm[1]);
             break;
           case OPS.setFont:
             const [fontName, fontSize] = args;
diff --git a/test/integration/freetext_editor_spec.mjs b/test/integration/freetext_editor_spec.mjs
index 6d0641de7..35f10c542 100644
--- a/test/integration/freetext_editor_spec.mjs
+++ b/test/integration/freetext_editor_spec.mjs
@@ -1799,6 +1799,83 @@ describe("FreeText Editor", () => {
     });
   });
 
+  describe("FreeText (open existing generated with Cairo)", () => {
+    let pages;
+
+    beforeEach(async () => {
+      pages = await loadAndWait(
+        "issue20504.pdf",
+        ".annotationEditorLayer",
+        100
+      );
+    });
+
+    afterEach(async () => {
+      await closePages(pages);
+    });
+
+    it("must open some existing annotations", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const boxes = [];
+          for (const num of [48, 49, 50, 51, 52]) {
+            const id = `${num}R`;
+            await page.waitForSelector(getAnnotationSelector(id), {
+              visible: true,
+            });
+            const rect = await getRect(page, getAnnotationSelector(id));
+            boxes.push(rect);
+          }
+
+          await switchToFreeText(page);
+
+          // The font sizes extracted from the Cairo appearance streams.
+          const expectedFontSizes = [61, 38, 38, 38, 56];
+          const PRECISION = 0.3;
+          // The width tracks the rendered text length, which depends heavily on
+          // the substitute font used to display the annotation (the original
+          // font isn't embedded). It can vary a lot from a platform to another,
+          // so we only check that it has the same order of magnitude as the
+          // annotation rect.
+          const WIDTH_PRECISION = 0.6;
+
+          for (let i = 0; i < boxes.length; i++) {
+            const rect = await getRect(page, `#pdfjs_internal_editor_${i}`);
+
+            // The default used font can be different from a platform to another
+            // hence we just check that the dimensions have the some order of
+            // magnitude as the annotation rect.
+            expect(Math.abs(rect.width / boxes[i].width - 1))
+              .withContext(`In ${browserName}, editor ${i} width`)
+              .toBeLessThan(WIDTH_PRECISION);
+            expect(Math.abs(rect.height / boxes[i].height - 1))
+              .withContext(`In ${browserName}, editor ${i} height`)
+              .toBeLessThan(PRECISION);
+            expect(Math.abs(rect.x / boxes[i].x - 1))
+              .withContext(`In ${browserName}, editor ${i} x`)
+              .toBeLessThan(PRECISION);
+            expect(Math.abs(rect.y / boxes[i].y - 1))
+              .withContext(`In ${browserName}, editor ${i} y`)
+              .toBeLessThan(PRECISION);
+
+            // Verify that the font size is correctly extracted from the Cairo
+            // appearance stream (font size is encoded in the cm operator).
+            const fontSize = await page.evaluate(N => {
+              const editorDiv = document.getElementById(
+                `pdfjs_internal_editor_${N}-editor`
+              );
+              const match = editorDiv?.style.fontSize.match(/calc\((\d+)px/);
+              return match ? parseInt(match[1], 10) : 0;
+            }, i);
+            expect(fontSize)
+              .withContext(`In ${browserName}, editor ${i} fontSize`)
+              .toEqual(expectedFontSizes[i]);
+          }
+        })
+      );
+    });
+  });
+
   describe("Keyboard shortcuts when the editor layer isn't focused", () => {
     let pages;
 
diff --git a/test/pdfs/.gitignore b/test/pdfs/.gitignore
index fdec2ee06..a7b875122 100644
--- a/test/pdfs/.gitignore
+++ b/test/pdfs/.gitignore
@@ -940,3 +940,4 @@
 !cff_bluescale_small_zones.pdf
 !multimedia_annotations.pdf
 !issue17333.pdf
+!issue20504.pdf
diff --git a/test/pdfs/issue20504.pdf b/test/pdfs/issue20504.pdf
new file mode 100644
index 000000000..7edea6800
Binary files /dev/null and b/test/pdfs/issue20504.pdf differ