From d35991a4686ff59f8f3f71d80c61b583644f675a Mon Sep 17 00:00:00 2001 From: Lucas Smith Date: Fri, 25 Sep 2026 12:29:39 +1000 Subject: [PATCH 1/3] fix(text): keep strings intact when extracting overlapping and layered text Lines are built from runs (strings drawn as a unit) that only share a line when their glyphs don't collide, instead of x-sorting every glyph on a baseline. Text in form XObjects is now extracted, and Tf/q-Q, rise, Tw and CMap decoding follow the spec. --- .agents/GOALS.md | 4 +- .agents/IMPLEMENTATION.md | 43 +- .../text/overlapping-invisible-labels.pdf | Bin 0 -> 1083 bytes src/api/pdf-page.ts | 77 +-- src/fonts/composite-font.test.ts | 31 ++ src/fonts/composite-font.ts | 15 +- src/fonts/embedded-font.ts | 13 +- src/fonts/pdf-font.ts | 14 + src/fonts/simple-font.test.ts | 12 + src/integration/text/text-extraction.test.ts | 40 ++ src/text/index.ts | 3 +- src/text/line-grouper.test.ts | 372 ++++++++++++++- src/text/line-grouper.ts | 437 ++++++++++++------ src/text/text-extractor.test.ts | 295 ++++++++++++ src/text/text-extractor.ts | 163 +++---- src/text/text-resources.ts | 128 +++++ src/text/text-state.ts | 173 +++---- 17 files changed, 1388 insertions(+), 432 deletions(-) create mode 100644 fixtures/text/overlapping-invisible-labels.pdf create mode 100644 src/text/text-extractor.test.ts create mode 100644 src/text/text-resources.ts diff --git a/.agents/GOALS.md b/.agents/GOALS.md index 4f5ab391..2d6004dc 100644 --- a/.agents/GOALS.md +++ b/.agents/GOALS.md @@ -56,8 +56,10 @@ This document captures the high-level goals for @libpdf/core. Use this to steer - [x] **Extract text** — Get text content from pages (extractText API) - [x] **Search text** — Find text patterns with bounding box results (findText API) -- [x] **Preserve reading order** — Line grouping based on baseline/font +- [x] **Preserve reading order** — Run-based line grouping (strings stay intact, overlaps split) +- [x] **Extract from form XObjects** — Headers, footers, embedded pages, flattened fields - [ ] **Extract from annotations** — Include comment text, form values +- [ ] **Respect clipping** — Drop text outside the current clip path ### 9. Creation diff --git a/.agents/IMPLEMENTATION.md b/.agents/IMPLEMENTATION.md index d1855e6f..1ba878c9 100644 --- a/.agents/IMPLEMENTATION.md +++ b/.agents/IMPLEMENTATION.md @@ -248,20 +248,45 @@ Embedded file specification handling. Text extraction from PDF content streams with position tracking. -| Component | Purpose | -| --------------- | ------------------------------------------------ | -| `TextExtractor` | Parses content streams, tracks text state | -| `TextState` | Manages text matrix, font, and positioning | -| `LineGrouper` | Groups characters into lines based on baseline | -| `text-search` | String and regex search with bounding boxes | -| `types.ts` | TextChar, TextLine, TextSpan, SearchResult types | +| Component | Purpose | +| --------------- | --------------------------------------------------------------------- | +| `TextExtractor` | Interprets content streams and form XObjects, emits positioned glyphs | +| `TextResources` | Fonts and form XObjects reachable from a `/Resources` dict, cached | +| `TextState` | Graphics/text state; glyph geometry through Tm × CTM | +| `LineGrouper` | Groups glyphs into runs, runs into lines, lines into spans | +| `text-search` | String and regex search with bounding boxes | +| `types.ts` | ExtractedChar, TextLine, TextSpan, TextMatch types | ### Supported Text Operators - Positioning: `Td`, `TD`, `Tm`, `T*` -- Showing: `Tj`, `TJ`, `'`, `"` +- Showing: `Tj`, `TJ`, `'`, `"` (codes split via the font's CMap; `Tw` on single-byte code 32) - State: `Tf`, `Tc`, `Tw`, `Tz`, `TL`, `Ts`, `Tr` -- Graphics: `cm`, `q`, `Q` (matrix transformations) +- Graphics: `cm`, `q`, `Q` (font and text state are part of the saved state) +- XObjects: `Do` for `/Subtype /Form` (own or inherited resources, `/Matrix`, cycle and depth guard) + +### Line Grouping Model + +Glyphs arrive in stream order. Consecutive glyphs that advance along one baseline form a +**run** (the string a producer drew as a unit). Runs are placed on lines: a run joins a +line if it shares the baseline and none of its space-occupying glyphs collide with glyphs +already there; otherwise it opens a new line on the same baseline. This keeps strings drawn +over each other (tags over invisible labels, layered text) intact instead of interleaving +them glyph by glyph. Within a line, runs are k-way merged by glyph x while intra-run order +is preserved. + +Rules layered on that model, each with a fixture or unit test: + +- Whitespace and overlay glyphs (combining marks, accents) do not occupy space. +- A glyph repeating one already on the line at the same position is dropped (fake bold, + redrawn chunk boundaries). +- Whitespace-only runs stay with the run drawn before them. +- Lines whose glyphs predominantly move left in stream order keep stream order (design-tool + exports, RTL). + +Known limitations: rotated/vertical text, multi-column pages sharing a baseline, an +invisible text layer that duplicates visible text at _different_ positions (yields a +duplicate line), text clipped away by `W n`, and `Tr 3` text is extracted like any other. ## Annotations Layer (`src/annotations/`) diff --git a/fixtures/text/overlapping-invisible-labels.pdf b/fixtures/text/overlapping-invisible-labels.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5d2c92239ec8ce54e0c82dab6e92f6e03eb9fb12 GIT binary patch literal 1083 zcmZWo%Z}496y5tP?xqq*G=9Z#Dvd^>&<+wHM$Hzl)`y(zu z&xF@2Ew(ifIs*a#^v*QZzTRL)7k*ixh+l9}t_Q7wE)Mo)Wm9WBbhARq9z*!JE2r9r z`(Vo4eua?x^o^}e`bg#pp-T>!C)t9=RE$BA!HZRqm#{Xw>c>k+9>^jYIuYLjb(VQU z1lJ%X3$*gAZDDp9&oGx~i70f)DJ%wf8VVgKn932iLEo0EJ*`dSb;W?4r*W6fpw5Yq zJdlyXNHPdno9XGv8NRCd^K%y}cBUc_r?}{F9?Vo2yW?aagGg!a4iR%af{24qF%C0~ z##K*uu~NIQM;@rbqF8qtdd8!6jT#*H6^9V(QNWQxp8~gtgV|kDH=^$P`^#)++7Aa^ z?bp_ixGQp?^RzK7^3LqO8(VBroS1S`Wkree|G6k%mrXGgj?>~ceQI!=Gm0W?G+Bq* zYzYHR*ziH5lvJ>Vp~f+?`=C75B0SEs8W(tjXEmYYi<*|Ri&`u(bw6;Eb95XVREIuvCe|<_D+W-In literal 0 HcmV?d00001 diff --git a/src/api/pdf-page.ts b/src/api/pdf-page.ts index 628c15c7..1d8dc51a 100644 --- a/src/api/pdf-page.ts +++ b/src/api/pdf-page.ts @@ -91,15 +91,12 @@ import type { } from "#src/drawing/types"; import { resolveRotationOrigin } from "#src/drawing/types"; import { EmbeddedFont } from "#src/fonts/embedded-font"; -import { parseFont } from "#src/fonts/font-factory"; -import type { PdfFont } from "#src/fonts/pdf-font"; import { getEncodingForStandard14, getStandard14BasicMetrics, isStandard14Font, isWinAnsiStandard14, } from "#src/fonts/standard-14"; -import { parseToUnicode } from "#src/fonts/to-unicode"; // Annotation utilities - imported here to avoid dynamic require issues import { concatBytes } from "#src/helpers/buffer"; import { black } from "#src/helpers/colors"; @@ -127,6 +124,7 @@ import { PdfStream } from "#src/objects/pdf-stream"; import { PdfString } from "#src/objects/pdf-string"; import { getPlainText, groupCharsIntoLines } from "#src/text/line-grouper"; import { TextExtractor } from "#src/text/text-extractor"; +import { TextResources } from "#src/text/text-resources"; import { searchPage } from "#src/text/text-search"; import type { ExtractTextOptions, FindTextOptions, PageText, TextMatch } from "#src/text/types"; @@ -2817,12 +2815,12 @@ export class PDFPage { // Get content stream bytes const contentBytes = this.getContentBytes(); - // Create font resolver - const resolveFont = this.createFontResolver(); - // Extract characters - const extractor = new TextExtractor({ resolveFont }); - const chars = extractor.extract(contentBytes); + const resources = new TextResources( + this.resolveInheritedResources(), + this.ctx.resolve.bind(this.ctx), + ); + const chars = new TextExtractor(resources).extract(contentBytes); // Group into lines and spans const lines = groupCharsIntoLines(chars); @@ -2936,8 +2934,11 @@ export class PDFPage { private resolveInheritedResources(): PdfDict | null { // Start with the page dict let currentDict: PdfDict | null = this.dict; + const visited = new Set(); + + while (currentDict && !visited.has(currentDict)) { + visited.add(currentDict); - while (currentDict) { // Check for Resources on the current node const resources = currentDict.get("Resources", this.ctx.resolve.bind(this.ctx)); @@ -2957,62 +2958,4 @@ export class PDFPage { return null; } - - /** - * Create a font resolver function for text extraction. - */ - private createFontResolver(): (name: string) => PdfFont | null { - // Get the page's Font resources (may be a ref or inherited from parent) - const resourcesDict = this.resolveInheritedResources(); - - if (!resourcesDict) { - return () => null; - } - - const font = resourcesDict.getDict("Font", this.ctx.resolve.bind(this.ctx)); - - if (!font) { - return () => null; - } - - // Preload all font dictionaries and build the cache - const fontCache = new Map(); - - for (const [key, entry] of font) { - const name = key.value; - const resolved = entry instanceof PdfRef ? this.ctx.resolve(entry) : entry; - - let entryDict = resolved instanceof PdfDict ? resolved : null; - - if (!entryDict) { - continue; - } - - // Parse ToUnicode CMap if present - let toUnicodeMap = null; - - const toUnicode = entryDict.get("ToUnicode", this.ctx.resolve.bind(this.ctx)); - const toUnicodeStream = toUnicode instanceof PdfStream ? toUnicode : null; - - if (toUnicodeStream) { - try { - toUnicodeMap = parseToUnicode(toUnicodeStream.getDecodedData()); - } catch { - // ToUnicode parsing failed - continue without it - } - } - - // Parse the font - const pdfFont = parseFont(entryDict, { - resolver: this.ctx.resolve.bind(this.ctx), - toUnicodeMap, - }); - - fontCache.set(name, pdfFont); - } - - return (name: string): PdfFont | null => { - return fontCache.get(name) ?? null; - }; - } } diff --git a/src/fonts/composite-font.test.ts b/src/fonts/composite-font.test.ts index beb09455..909458af 100644 --- a/src/fonts/composite-font.test.ts +++ b/src/fonts/composite-font.test.ts @@ -151,6 +151,37 @@ describe("CompositeFont", () => { }); }); + describe("decode", () => { + const cidFont = new CIDFont({ subtype: "CIDFontType2", baseFontName: "TestFont" }); + + it("reads two bytes per code for Identity-H", () => { + const font = new CompositeFont({ baseFontName: "TestFont", cmap: CMap.identityH(), cidFont }); + + expect(font.decode(new Uint8Array([0x00, 0x41, 0x4e, 0x2d]))).toEqual([ + { code: 0x41, length: 2 }, + { code: 0x4e2d, length: 2 }, + ]); + }); + + it("follows the CMap codespace ranges for mixed-width codes", () => { + // Shift-JIS style: single-byte ASCII, double-byte kanji + const cmap = new CMap({ + name: "Custom", + codespaceRanges: [ + { low: 0x20, high: 0x7e, numBytes: 1 }, + { low: 0x8140, high: 0x9ffc, numBytes: 2 }, + ], + }); + const font = new CompositeFont({ baseFontName: "TestFont", cmap, cidFont }); + + expect(font.decode(new Uint8Array([0x41, 0x81, 0x40, 0x20]))).toEqual([ + { code: 0x41, length: 1 }, + { code: 0x8140, length: 2 }, + { code: 0x20, length: 1 }, + ]); + }); + }); + describe("canEncode", () => { it("should return true for Identity-H", () => { const cmap = CMap.identityH(); diff --git a/src/fonts/composite-font.ts b/src/fonts/composite-font.ts index 8d968995..525362de 100644 --- a/src/fonts/composite-font.ts +++ b/src/fonts/composite-font.ts @@ -28,7 +28,7 @@ import { PdfStream } from "#src/objects/pdf-stream.ts"; import { CIDFont, parseCIDFont } from "./cid-font"; import { CMap, parseCMap } from "./cmap"; import type { FontDescriptor } from "./font-descriptor"; -import { PdfFont } from "./pdf-font"; +import { type CharCode, PdfFont } from "./pdf-font"; import type { ToUnicodeMap } from "./to-unicode"; /** @@ -101,6 +101,19 @@ export class CompositeFont extends PdfFont { return this.cmap.encode(text); } + /** Codes are one to four bytes, per the CMap's codespace ranges. */ + decode(bytes: Uint8Array): CharCode[] { + const codes: CharCode[] = []; + + for (let offset = 0; offset < bytes.length; ) { + const next = this.cmap.readCharCode(bytes, offset); + codes.push(next); + offset += next.length; + } + + return codes; + } + /** * Decode character code to Unicode string (for text extraction). */ diff --git a/src/fonts/embedded-font.ts b/src/fonts/embedded-font.ts index 104d1231..3bf3f206 100644 --- a/src/fonts/embedded-font.ts +++ b/src/fonts/embedded-font.ts @@ -14,7 +14,7 @@ import type { PdfRef } from "#src/objects/pdf-ref.ts"; import { parseFontProgram } from "./embedded-parser.ts"; import { FontDescriptor } from "./font-descriptor.ts"; import { type FontProgram, TrueTypeFontProgram } from "./font-program/index.ts"; -import { PdfFont } from "./pdf-font.ts"; +import { type CharCode, PdfFont } from "./pdf-font.ts"; /** * Options for embedding a font. @@ -239,6 +239,17 @@ export class EmbeddedFont extends PdfFont { return this.trackAndEncode(text).map(e => e.codePoint); } + /** Identity-H: two bytes per code. */ + decode(bytes: Uint8Array): CharCode[] { + const codes: CharCode[] = []; + + for (let offset = 0; offset + 1 < bytes.length; offset += 2) { + codes.push({ code: (bytes[offset] << 8) | bytes[offset + 1], length: 2 }); + } + + return codes; + } + /** * Encode text to glyph IDs for PDF content stream. * diff --git a/src/fonts/pdf-font.ts b/src/fonts/pdf-font.ts index 20445e88..6935ff11 100644 --- a/src/fonts/pdf-font.ts +++ b/src/fonts/pdf-font.ts @@ -13,6 +13,12 @@ import type { FontDescriptor } from "./font-descriptor"; +/** A character code and the number of bytes it occupied in the string. */ +export interface CharCode { + code: number; + length: number; +} + /** * Abstract base class for PDF fonts. */ @@ -58,6 +64,14 @@ export abstract class PdfFont { */ abstract toUnicode(code: number): string; + /** + * Split string operand bytes into character codes. One byte per code + * unless overridden. + */ + decode(bytes: Uint8Array): CharCode[] { + return Array.from(bytes, code => ({ code, length: 1 })); + } + /** * Check if this font can encode the given text. * diff --git a/src/fonts/simple-font.test.ts b/src/fonts/simple-font.test.ts index b766b758..1764d248 100644 --- a/src/fonts/simple-font.test.ts +++ b/src/fonts/simple-font.test.ts @@ -40,6 +40,18 @@ describe("SimpleFont", () => { }); }); + describe("decode", () => { + it("reads one byte per code", () => { + const font = new SimpleFont({ subtype: "TrueType", baseFontName: "Helvetica" }); + + expect(font.decode(new Uint8Array([0x48, 0x20, 0xe9]))).toEqual([ + { code: 0x48, length: 1 }, + { code: 0x20, length: 1 }, + { code: 0xe9, length: 1 }, + ]); + }); + }); + describe("getWidth", () => { it("should return width from widths array", () => { const font = new SimpleFont({ diff --git a/src/integration/text/text-extraction.test.ts b/src/integration/text/text-extraction.test.ts index 65ff72b8..8c6a0b39 100644 --- a/src/integration/text/text-extraction.test.ts +++ b/src/integration/text/text-extraction.test.ts @@ -222,6 +222,46 @@ describe("Text Extraction Integration", () => { }); }); + describe("template tags drawn over invisible labels", () => { + // Field labels drawn with /ca 0 and the tag printed on top, same baseline + it("finds every tag intact", async () => { + const bytes = await loadFixture("text", "overlapping-invisible-labels.pdf"); + const pdf = await PDF.load(bytes); + const page = pdf.getPage(0); + + const matches = page!.findText(/\{\{[^}]*\}\}/g).map(m => m.text); + + expect(matches).toEqual([ + "{{date,r1}}", + "{{signature,r1}}", + "{{signature,r2}}", + "{{date,r2}}", + ]); + }); + + it("keeps the overlapped label on its own line", async () => { + const bytes = await loadFixture("text", "overlapping-invisible-labels.pdf"); + const pdf = await PDF.load(bytes); + const page = pdf.getPage(0); + + const lines = page!.extractText().lines.map(l => l.text); + + expect(lines).toContain("{{date,r2}}"); + expect(lines.some(l => l.includes("field2date"))).toBe(true); + expect(lines.some(l => l.includes("field2date") && l.includes("{{date,r2}}"))).toBe(false); + }); + }); + + describe("malformed documents", () => { + it("terminates on a page tree with a Parent cycle", async () => { + const bytes = await loadFixture("malformed", "PDFBOX-6040-nodeloop.pdf"); + const pdf = await PDF.load(bytes); + const page = pdf.getPage(0); + + expect(() => page!.extractText()).not.toThrow(); + }); + }); + describe("bounding boxes", () => { it("provides valid bounding boxes", async () => { const bytes = await loadFixture("text", "rot0.pdf"); diff --git a/src/text/index.ts b/src/text/index.ts index e7f2703a..b2e3fd84 100644 --- a/src/text/index.ts +++ b/src/text/index.ts @@ -6,7 +6,8 @@ */ export { getPlainText, groupCharsIntoLines, type LineGrouperOptions } from "./line-grouper"; -export { TextExtractor, type TextExtractorOptions } from "./text-extractor"; +export { TextExtractor } from "./text-extractor"; +export { type FormXObject, TextResources } from "./text-resources"; export { searchPage, searchPages } from "./text-search"; export { TextState } from "./text-state"; export * from "./types"; diff --git a/src/text/line-grouper.test.ts b/src/text/line-grouper.test.ts index ba764566..4a6ae71a 100644 --- a/src/text/line-grouper.test.ts +++ b/src/text/line-grouper.test.ts @@ -337,13 +337,14 @@ describe("LineGrouper", () => { it("detects RTL-placed at exactly 80% threshold", () => { // 6 chars → 5 pairs. 4 decreasing = 80% → should be detected. // Adjacent chars (width=8) so gaps are 0 and no spaces inserted. + // The forward jump lands in free space so it doesn't overlap a neighbour. const chars = [ makeChar("A", 50, 0), makeChar("B", 42, 1), // decreasing makeChar("C", 34, 2), // decreasing makeChar("D", 26, 3), // decreasing - makeChar("E", 28, 4), // increasing (forward jump) - makeChar("F", 20, 5), // decreasing + makeChar("E", 18, 4), // decreasing + makeChar("F", 60, 5), // increasing (forward jump) ]; const lines = groupCharsIntoLines(chars); @@ -354,19 +355,20 @@ describe("LineGrouper", () => { it("falls back to x-sort below 80% threshold", () => { // 6 chars → 5 pairs. 3 decreasing = 60% → NOT detected → x-sort. + // Every char jumps to a new position, so each is its own run; none overlap. const chars = [ - makeChar("A", 50, 0), - makeChar("B", 42, 1), // decreasing - makeChar("C", 44, 2), // increasing - makeChar("D", 36, 3), // decreasing - makeChar("E", 38, 4), // increasing - makeChar("F", 30, 5), // decreasing + makeChar("A", 40, 0), + makeChar("B", 24, 1), // decreasing + makeChar("C", 32, 2), // increasing + makeChar("D", 8, 3), // decreasing + makeChar("E", 16, 4), // increasing + makeChar("F", 0, 5), // decreasing ]; const lines = groupCharsIntoLines(chars); expect(lines).toHaveLength(1); - // x-sorted order: F(30), D(36), E(38), B(42), C(44), A(50) + // x-sorted order: F(0), D(8), E(16), B(24), C(32), A(40) expect(lines[0].text).toBe("FDEBCA"); }); @@ -501,8 +503,7 @@ describe("LineGrouper", () => { expect(lines[1].baseline).toBe(80); }); - it("falls back to x-sort when sequenceIndex is missing", () => { - // Chars placed right-to-left but without sequenceIndex — should x-sort + it("treats array order as stream order even without sequenceIndex", () => { const chars = [ makeChar("A", 30, undefined), makeChar("B", 20, undefined), @@ -513,8 +514,353 @@ describe("LineGrouper", () => { const lines = groupCharsIntoLines(chars); expect(lines).toHaveLength(1); - // x-sort produces D(0), C(10), B(20), A(30) - expect(lines[0].text).toBe("DCBA"); + expect(lines[0].text).toBe("ABCD"); + }); + }); + + describe("run-based line assembly", () => { + function makeChar( + char: string, + x: number, + options: { baseline?: number; width?: number; fontSize?: number } = {}, + ): ExtractedChar { + const { baseline = 10, width = 6, fontSize = 12 } = options; + + return { + char, + bbox: { x, y: baseline - 2, width, height: fontSize }, + fontSize, + fontName: "Helvetica", + baseline, + }; + } + + /** Lay out a string as adjacent glyphs starting at x. */ + function makeRun(text: string, x: number, options: Parameters[2] = {}) { + const width = options.width ?? 6; + + return Array.from(text, (char, i) => makeChar(char, x + i * width, options)); + } + + it("keeps a string intact when another string is drawn over it", () => { + // Invisible field label under a template tag; x-sorting gave "winte{{xd2adtea,tre2}}" + const label = makeRun("wintex2date", 482); + const tag = makeRun("{{date,r2}}", 500); + + const lines = groupCharsIntoLines([...label, ...tag]); + + expect(lines).toHaveLength(2); + expect(lines.map(l => l.text)).toEqual(["wintex2date", "{{date,r2}}"]); + }); + + it("keeps glyph order inside a run even if x-positions are not monotonic", () => { + const chars = [ + makeChar("V", 10, { width: 8 }), + makeChar("A", 17, { width: 8 }), // 1pt overlap with V — still the same run + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("VA"); + }); + + it("orders non-overlapping runs left to right regardless of draw order", () => { + const number = makeRun("42", 500); + const title = makeRun("Title", 50); + + const lines = groupCharsIntoLines([...number, ...title]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("Title 42"); + }); + + it("splits a run at a large forward jump so other text can sit in the gap", () => { + // One TJ with a large jump; a value is drawn into the gap later + const labels = [...makeRun("Name:", 0), ...makeRun("Date:", 200)]; + const value = makeRun("Alice", 40); + + const lines = groupCharsIntoLines([...labels, ...value]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("Name: Alice Date:"); + }); + + it("starts a new line for a run that overlaps only part of an existing line", () => { + const first = makeRun("Hello", 0); + const overlapping = makeRun("World", 20); + const after = makeRun("!", 100); + + const lines = groupCharsIntoLines([...first, ...overlapping, ...after]); + + expect(lines.map(l => l.text)).toEqual(["Hello !", "World"]); + }); + + it("tolerates small overlaps between adjacent runs", () => { + const first = makeRun("fi", 0, { width: 6 }); + const second = makeRun("x", 11); + + const lines = groupCharsIntoLines([...first, ...second]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("fix"); + }); + + it("lets a run fill gaps inside another run", () => { + // "l" and "i" drawn in a second pass into gaps left by the first + const wide = [ + makeChar("F", 0), + makeChar("o", 6), + makeChar("a", 18), + makeChar("t", 24), + makeChar("o", 36), + makeChar("n", 42), + ]; + const narrow = [makeChar("l", 12, { width: 3 }), makeChar("i", 15, { width: 3 })]; + const dot = [makeChar("i", 30)]; + + const lines = groupCharsIntoLines([...wide, ...narrow, ...dot]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("Foliation"); + }); + + it("continues a run through CJK punctuation compression", () => { + // Full-width "、" advances 1em; the next glyph is pulled in exactly 0.5em + const chars = [ + makeChar("人", 0, { width: 10.56, fontSize: 10.56 }), + makeChar("、", 10.56, { width: 10.56, fontSize: 10.56 }), + makeChar("法", 15.84, { width: 10.56, fontSize: 10.56 }), + makeChar("人", 26.4, { width: 10.56, fontSize: 10.56 }), + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("人、法人"); + }); + + it("ignores whitespace glyphs when checking for overlap", () => { + // Space glyph kerned back over a ligature (AGaramond "fi ve") + const chars = [ + makeChar("fi", 0, { width: 5 }), + makeChar(" ", 2.5, { width: 2.4 }), // starts 2.5pt back — new run, no ink + makeChar("v", 5, { width: 4 }), + makeChar("e", 9, { width: 4 }), + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("fi ve"); + }); + + it("assigns an overlapping run to the first line where it fits", () => { + // A (0–24) and B (12–36) collide; C (6–18) collides with both; D (50–62) fits on A's line + const a = makeRun("AAAA", 0); + const b = makeRun("BBBB", 12); + const c = makeRun("CC", 6); + const d = makeRun("DD", 50); + + const lines = groupCharsIntoLines([...a, ...b, ...c, ...d]); + + expect(lines.map(l => l.text)).toEqual(["AAAA DD", "BBBB", "CC"]); + }); + + it("does not treat zero-width glyphs as overlapping", () => { + const chars = [ + makeChar("A", 30, { width: 0 }), + makeChar("B", 20, { width: 0 }), + makeChar("C", 10, { width: 0 }), + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines).toHaveLength(1); + expect(lines[0].spans.flatMap(s => s.chars).filter(c => c.char !== " ")).toHaveLength(3); + }); + + it("anchors a line on its first visible glyph, not leading spaces", () => { + // Leading spaces sit 2pt below the letters; "Total" is 1.2pt above them + const spaces = [makeChar(" ", 0, { baseline: 8 }), makeChar(" ", 6, { baseline: 8 })]; + const label = makeRun("Acrobatics", 12, { baseline: 10 }); + const total = makeRun("Total", 120, { baseline: 11.2 }); + + const lines = groupCharsIntoLines([...spaces, ...label, ...total]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe(" Acrobatics Total"); + }); + + it("inserts a space at a gap that coincides with a font change", () => { + const chars = [ + makeChar("P", 0, { fontSize: 4.5 }), + ...makeRun("SPELL", 40, { fontSize: 6.5 }), + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe("P SPELL"); + expect(lines[0].spans).toHaveLength(2); + expect(lines[0].spans[0].text).toBe("P "); + }); + + it("does not synthesise a space next to a real space glyph", () => { + const chars = [ + ...makeRun("1.", 0), + makeChar(" ", 12, { width: 3 }), + ...makeRun("Lorem", 30, { fontSize: 14 }), + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines[0].text).toBe("1. Lorem"); + }); + + it("detects a glyph straddling two glyphs of another run", () => { + // "s" covers 3pt of "a" and 3pt of "b": neither pair alone exceeds tolerance + const first = makeRun("ab", 0); + const second = [makeChar("s", 3)]; + + const lines = groupCharsIntoLines([...first, ...second]); + + expect(lines.map(l => l.text)).toEqual(["ab", "s"]); + }); + + it("detects a narrow glyph fully covered by a wider one", () => { + const wide = makeRun("mm", 0, { width: 10 }); + const narrow = [makeChar("i", 4, { width: 2 })]; + + const lines = groupCharsIntoLines([...wide, ...narrow]); + + expect(lines.map(l => l.text)).toEqual(["mm", "i"]); + }); + + it("keeps an ink-less run with the run drawn before it", () => { + // Invisible per-glyph layer placed mirrored over visible text; its space + // glyph must not land inside the visible words + const visible = makeRun("ab cd ef", 0); + const layer = [ + makeChar("a", 24), + makeChar("b", 18), + makeChar(" ", 12), + makeChar("c", 6), + makeChar("d", 0), + ]; + + const lines = groupCharsIntoLines([...visible, ...layer]); + + expect(lines[0].text).toBe("ab cd ef"); + }); + + it("preserves stream order for equal-baseline lines", () => { + const label = makeRun("label", 100); + const tag = makeRun("{{tag}}", 100); + + const lines = groupCharsIntoLines([...label, ...tag]); + + expect(lines.map(l => l.text)).toEqual(["label", "{{tag}}"]); + }); + }); + + describe("repeated glyphs", () => { + function makeChar( + char: string, + x: number, + options: { width?: number; baseline?: number } = {}, + ) { + const { width = 6, baseline = 10 } = options; + + return { + char, + bbox: { x, y: baseline - 2, width, height: 12 }, + fontSize: 12, + fontName: "Helvetica", + baseline, + } satisfies ExtractedChar; + } + + function makeRun(text: string, x: number, options: Parameters[2] = {}) { + const width = options.width ?? 6; + + return Array.from(text, (char, i) => makeChar(char, x + i * width, options)); + } + + it("collapses fake bold (same text drawn twice with a small offset)", () => { + const lines = groupCharsIntoLines([...makeRun("Bold", 0), ...makeRun("Bold", 0.3)]); + + expect(lines.map(l => l.text)).toEqual(["Bold"]); + }); + + it("collapses a partial redraw inside a longer line", () => { + const line = makeRun("the 11th Amendment.", 0); + const bold = makeRun("Amendment", 9 * 6 + 0.3); + + const lines = groupCharsIntoLines([...line, ...bold]); + + expect(lines.map(l => l.text)).toEqual(["the 11th Amendment."]); + }); + + it("drops the redrawn boundary glyph between paragraph chunks", () => { + // Chunks scattered through the stream, each redrawing the previous + // chunk's last glyph at a slightly different x + const chunk1 = makeRun("By sign", 0); + const other = makeRun("elsewhere", 0, { baseline: 50 }); + const chunk2 = makeRun("ning below", 6 * 6 - 0.12); + + const lines = groupCharsIntoLines([...chunk1, ...other, ...chunk2]); + + expect(lines.map(l => l.text)).toEqual(["elsewhere", "By signing below"]); + }); + + it("drops only the repeated glyphs and keeps the rest of the run", () => { + const first = makeRun("abc", 0); + const second = [makeChar("c", 12.2), makeChar("d", 18), makeChar("e", 24)]; + + const lines = groupCharsIntoLines([...first, ...second]); + + expect(lines.map(l => l.text)).toEqual(["abcde"]); + }); + + it("treats a different character at the same position as a collision", () => { + const lines = groupCharsIntoLines([...makeRun("O", 0), ...makeRun("/", 0.2)]); + + expect(lines.map(l => l.text)).toEqual(["O", "/"]); + }); + + it("lets an accent drawn over its base share the line", () => { + // OT1 TeX: accent first, then the base glyph drawn underneath it + const chars = [ + ...makeRun("caf", 0), + makeChar("\u00B4", 18.5, { width: 3 }), + makeChar("e", 18), + ]; + + const lines = groupCharsIntoLines(chars); + + expect(lines.map(l => l.text)).toEqual(["cafe\u00B4"]); + }); + + it("keeps identical text that does not overlap", () => { + const lines = groupCharsIntoLines([...makeRun("0", 0), ...makeRun("0", 100)]); + + expect(lines.map(l => l.text)).toEqual(["0 0"]); + }); + + it("keeps overlapping lines whose text differs", () => { + const lines = groupCharsIntoLines([...makeRun("label", 0), ...makeRun("{{tag}}", 0)]); + + expect(lines.map(l => l.text)).toEqual(["label", "{{tag}}"]); + }); + + it("keeps identical text on different baselines", () => { + const lines = groupCharsIntoLines([ + ...makeRun("Title", 0, { baseline: 100 }), + ...makeRun("Title", 0, { baseline: 80 }), + ]); + + expect(lines.map(l => l.text)).toEqual(["Title", "Title"]); }); }); diff --git a/src/text/line-grouper.ts b/src/text/line-grouper.ts index a8c27e75..af8214b2 100644 --- a/src/text/line-grouper.ts +++ b/src/text/line-grouper.ts @@ -1,31 +1,71 @@ /** * LineGrouper - Groups extracted characters into lines and spans. * - * Characters are grouped into lines based on their baseline Y coordinate, - * and within lines into spans based on font/size changes. - * Spaces are detected from gaps between characters. + * Characters arrive in content-stream order and pass through three steps: + * + * 1. **Runs.** Consecutive characters advancing along one baseline form a + * run: the string a producer drew as a unit (as in pdf.js text items). + * 2. **Lines.** A run joins a line if it shares the baseline and none of its + * glyphs collide with glyphs already there; otherwise it opens a new line + * on the same baseline. Strings drawn over each other (a tag printed on an + * invisible label) stay intact instead of interleaving glyph by glyph. A + * glyph repeating one already on the line at the same position is dropped. + * 3. **Spans.** Runs are merged left to right by glyph position (stream order + * for RTL-placed text) and split into spans on font changes, with spaces + * synthesised from gaps. + * + * Whitespace and overlay glyphs (combining marks, accents) occupy no space and + * take no part in collision or repeat checks. Only horizontal text is handled. */ import type { ExtractedChar, TextLine, TextSpan } from "./types"; import { mergeBboxes } from "./types"; /** - * Minimum fraction of consecutive char pairs with decreasing x-positions - * (in stream order) to classify a line as "RTL-placed". - * - * Figma/Canva exports produce ~100% decreasing pairs within words. - * 80% tolerates small forward jumps at word boundaries. + * Run continuity, as fractions of the font size. A glyph may start up to + * RUN_BACKWARD before the previous glyph's right edge (CJK punctuation + * compression is exactly 0.5em, so the limit sits above it) and up to + * RUN_FORWARD after it, keeping runs at word granularity. */ +const RUN_BACKWARD_FACTOR = 0.6; +const RUN_FORWARD_FACTOR = 0.6; + +/** + * A glyph collides with another run once that run covers more than a quarter + * em of it, or 30% of its own width for narrow glyphs. Less is kerning or + * italic overhang. + */ +const GLYPH_COVER_EM_FACTOR = 0.25; +const GLYPH_COVER_WIDTH_FACTOR = 0.3; + +/** Same character within this fraction of an em is the same glyph drawn twice. */ +const REPEAT_POSITION_FACTOR = 0.1; + +/** Combining marks and modifier symbols (spacing accents) sit over other glyphs. */ +const OVERLAY_CHARS = /^[\p{Mn}\p{Mc}\p{Me}\p{Sk}]+$/u; + +/** Fraction of stream-order pairs moving left for a line to count as RTL-placed. */ const RTL_PLACED_THRESHOLD = 0.8; /** - * Result of ordering characters within a line. + * A maximal sequence of characters drawn continuously along one baseline. */ -interface OrderedLine { - /** Characters in reading order */ +interface TextRun { chars: ExtractedChar[]; - /** Whether the line was detected as RTL-placed (design-tool pattern) */ - rtlPlaced: boolean; + /** Baseline of the first space-occupying glyph; spaces are often placed oddly */ + baseline: number; + /** Extent of space-occupying glyphs; empty (left > right) if there are none */ + inkLeft: number; + inkRight: number; +} + +/** + * A line under construction. Baseline is that of the first run, not an + * average, so placement does not drift. + */ +interface LineBuilder { + baseline: number; + runs: TextRun[]; } /** @@ -50,9 +90,9 @@ export interface LineGrouperOptions { /** * Group extracted characters into lines and spans. * - * @param chars - Array of extracted characters + * @param chars - Extracted characters in content-stream order * @param options - Grouping options - * @returns Array of text lines + * @returns Array of text lines, top to bottom */ export function groupCharsIntoLines( chars: ExtractedChar[], @@ -65,159 +105,301 @@ export function groupCharsIntoLines( const baselineTolerance = options.baselineTolerance ?? 2; const spaceThreshold = options.spaceThreshold ?? 0.3; - // Group characters by baseline - const lineGroups = groupByBaseline(chars, baselineTolerance); + const runs = splitIntoRuns(chars, baselineTolerance); + const builders = placeRunsOnLines(runs, baselineTolerance); - // Convert each group to a TextLine const lines: TextLine[] = []; - for (const group of lineGroups) { - // Order characters within the line. - // Normally we sort left-to-right by x-position, but some design tools - // (Figma, Canva) place characters right-to-left via TJ adjustments while - // the text is actually LTR. In that case, content stream order is correct - // and position-based sorting would reverse the text. - const { chars: sorted, rtlPlaced } = orderLineChars(group); - - // Group into spans and detect spaces - const spans = groupIntoSpans(sorted, spaceThreshold, rtlPlaced); + for (const builder of builders) { + const { chars: ordered, rtlPlaced } = orderRuns(builder.runs); + const spans = groupIntoSpans(ordered, spaceThreshold, rtlPlaced); if (spans.length === 0) { continue; } - // Build the line - const lineText = spans.map(s => s.text).join(""); - const lineBbox = mergeBboxes(spans.map(s => s.bbox)); - const baseline = calculateAverageBaseline(sorted); - lines.push({ - text: lineText, - bbox: lineBbox, + text: spans.map(s => s.text).join(""), + bbox: mergeBboxes(spans.map(s => s.bbox)), spans, - baseline, + baseline: averageBaseline(ordered), }); } - // Sort lines top-to-bottom (higher Y = higher on page in PDF coordinates) + // Top to bottom; stable, so lines sharing a baseline keep stream order lines.sort((a, b) => b.baseline - a.baseline); return lines; } +function splitIntoRuns(chars: ExtractedChar[], baselineTolerance: number): TextRun[] { + const groups: ExtractedChar[][] = []; + let current: ExtractedChar[] = []; + + for (const char of chars) { + const prev = current[current.length - 1]; + + if (prev && !continuesRun(prev, char, baselineTolerance)) { + groups.push(current); + current = []; + } + + current.push(char); + } + + if (current.length > 0) { + groups.push(current); + } + + return groups.map(createRun); +} + +function createRun(chars: ExtractedChar[]): TextRun { + const run: TextRun = { + chars, + baseline: chars[0].baseline, + inkLeft: Infinity, + inkRight: -Infinity, + }; + + for (const char of chars) { + if (occupiesSpace(char)) { + if (!hasInk(run)) { + run.baseline = char.baseline; + } + + run.inkLeft = Math.min(run.inkLeft, char.bbox.x); + run.inkRight = Math.max(run.inkRight, char.bbox.x + char.bbox.width); + } + } + + return run; +} + +function hasInk(run: TextRun): boolean { + return run.inkLeft <= run.inkRight; +} + +function isWhitespace(char: ExtractedChar): boolean { + return char.char.trim() === ""; +} + +function occupiesSpace(char: ExtractedChar): boolean { + return !isWhitespace(char) && !OVERLAY_CHARS.test(char.char); +} + /** - * Determine the correct character order for a line. - * - * Design tools like Figma and Canva export PDFs where LTR characters are placed - * right-to-left via TJ positioning adjustments (positive values move the pen left). - * The font has near-zero glyph widths, so all positioning comes from TJ. Characters - * appear in correct reading order in the content stream, but their x-positions - * decrease monotonically. - * - * When this pattern is detected, we preserve content stream order instead of sorting - * by x-position, which would reverse the text. - * - * **Limitation**: Detection requires `sequenceIndex` on every character. If any - * character in the group lacks a `sequenceIndex`, we fall back to x-position sorting - * because stream order cannot be reliably reconstructed. + * Same baseline, and the pen advanced past the previous origin without a + * large jump either way. A glyph drawn on top of the previous one is a + * redraw, not a continuation. */ -function orderLineChars(group: ExtractedChar[]): OrderedLine { - if (group.length <= 1) { - return { chars: [...group], rtlPlaced: false }; +function continuesRun( + prev: ExtractedChar, + char: ExtractedChar, + baselineTolerance: number, +): boolean { + if (Math.abs(char.baseline - prev.baseline) > baselineTolerance) { + return false; } - // If any character lacks sequenceIndex, fall back to x-sort - const hasStreamOrder = group.every(c => c.sequenceIndex != null); + const fontSize = (prev.fontSize + char.fontSize) / 2; + const advanced = char.bbox.x - prev.bbox.x > fontSize * REPEAT_POSITION_FACTOR; + const gap = char.bbox.x - (prev.bbox.x + prev.bbox.width); + + return advanced && gap >= -fontSize * RUN_BACKWARD_FACTOR && gap <= fontSize * RUN_FORWARD_FACTOR; +} - if (!hasStreamOrder) { - return { - chars: [...group].sort((a, b) => a.bbox.x - b.bbox.x), - rtlPlaced: false, - }; +/** + * Each run joins the first line that shares its baseline and has room for + * it, or opens a new line. Repeated glyphs (fake bold, redrawn chunk + * boundaries) are dropped first. Whitespace-only runs cannot collide and + * stay with the run drawn before them. + */ +function placeRunsOnLines(runs: TextRun[], baselineTolerance: number): LineBuilder[] { + const lines: LineBuilder[] = []; + let previous: LineBuilder | null = null; + + for (const run of runs) { + previous = hasInk(run) + ? placeInkRun(run, lines, baselineTolerance) + : placeInklessRun(run, lines, previous, baselineTolerance); } - // Sort by sequenceIndex to get content stream order. - // Safe to use `!` — hasStreamOrder guarantees every char has sequenceIndex. - const streamOrder = [...group].sort((a, b) => a.sequenceIndex! - b.sequenceIndex!); + return lines; +} - if (isRtlPlaced(streamOrder)) { - return { chars: streamOrder, rtlPlaced: true }; +function placeInkRun(run: TextRun, lines: LineBuilder[], baselineTolerance: number): LineBuilder { + for (const line of lines) { + if (Math.abs(run.baseline - line.baseline) > baselineTolerance) { + continue; + } + + const fresh = withoutRepeatedGlyphs(run, line); + + if (!fresh || !hasInk(fresh)) { + return line; + } + + if (!line.runs.some(other => runsConflict(fresh, other))) { + line.runs.push(fresh); + + return line; + } } - // Normal case: sort left-to-right by x-position - return { - chars: [...group].sort((a, b) => a.bbox.x - b.bbox.x), - rtlPlaced: false, - }; + const line = { baseline: run.baseline, runs: [run] }; + lines.push(line); + + return line; +} + +function placeInklessRun( + run: TextRun, + lines: LineBuilder[], + previous: LineBuilder | null, + baselineTolerance: number, +): LineBuilder { + const sameBaseline = (line: LineBuilder) => + Math.abs(run.baseline - line.baseline) <= baselineTolerance; + + let line = previous && sameBaseline(previous) ? previous : lines.find(sameBaseline); + + if (!line) { + line = { baseline: run.baseline, runs: [] }; + lines.push(line); + } + + line.runs.push(run); + + return line; +} + +function withoutRepeatedGlyphs(run: TextRun, line: LineBuilder): TextRun | null { + const existing = line.runs.flatMap(r => r.chars); + const kept = run.chars.filter(glyph => !existing.some(other => isRepeatedGlyph(glyph, other))); + + if (kept.length === run.chars.length) { + return run; + } + + return kept.length > 0 ? createRun(kept) : null; +} + +function isRepeatedGlyph(glyph: ExtractedChar, other: ExtractedChar): boolean { + return ( + glyph.char === other.char && + Math.abs(glyph.bbox.x - other.bbox.x) <= glyph.fontSize * REPEAT_POSITION_FACTOR + ); } /** - * Detect whether characters are placed right-to-left in user space while - * content stream order represents the correct reading order. - * - * Returns true when x-positions in stream order are predominantly decreasing - * (≥ 80% of consecutive pairs). In that case, position-based sorting would - * reverse the reading order, so we preserve stream order instead. - * - * This covers two real-world scenarios: - * - **Design-tool PDFs** (Figma, Canva): LTR text placed right-to-left via - * TJ positioning adjustments. Stream order = correct reading order. - * - **Genuine RTL text** (Arabic, Hebrew): characters naturally placed - * right-to-left. PDF producers typically emit them in reading order, so - * stream order is again correct. - * - * In both cases, when x-positions decrease in stream order, preserving stream - * order produces the correct reading order. - * - * **Known limitation**: mixed bidi text (e.g., Arabic with embedded English) - * requires a full Unicode bidi algorithm, which is out of scope for this - * heuristic. For mixed lines, neither stream order nor x-sort is fully - * correct; a future bidi implementation should replace this heuristic. + * Checked per glyph so a run may fill gaps in another (some producers draw + * narrow glyphs in a second pass); coverage is summed so a glyph straddling + * two others is still caught. */ -function isRtlPlaced(streamOrder: ExtractedChar[]): boolean { - if (streamOrder.length < 2) { +function runsConflict(a: TextRun, b: TextRun): boolean { + if (overlapOf(a.inkLeft, a.inkRight, b.inkLeft, b.inkRight) <= 0) { return false; } - // Count how many consecutive character pairs have decreasing x - let decreasingCount = 0; - for (let i = 1; i < streamOrder.length; i++) { - if (streamOrder[i].bbox.x < streamOrder[i - 1].bbox.x) { - decreasingCount++; + return hasCoveredGlyph(a, b) || hasCoveredGlyph(b, a); +} + +function hasCoveredGlyph(run: TextRun, other: TextRun): boolean { + for (const glyph of run.chars) { + if (!occupiesSpace(glyph)) { + continue; + } + + const left = glyph.bbox.x; + const right = left + glyph.bbox.width; + const tolerance = Math.min( + glyph.fontSize * GLYPH_COVER_EM_FACTOR, + glyph.bbox.width * GLYPH_COVER_WIDTH_FACTOR, + ); + let covered = 0; + + for (const ink of other.chars) { + if (!occupiesSpace(ink)) { + continue; + } + + covered += Math.max(0, overlapOf(left, right, ink.bbox.x, ink.bbox.x + ink.bbox.width)); + + if (covered > tolerance) { + return true; + } } } - const totalPairs = streamOrder.length - 1; + return false; +} - return decreasingCount / totalPairs >= RTL_PLACED_THRESHOLD; +function overlapOf(aLeft: number, aRight: number, bLeft: number, bRight: number): number { + return Math.min(aRight, bRight) - Math.max(aLeft, bLeft); } /** - * Group characters by baseline Y coordinate. + * Flatten a line's runs into reading order: merged by glyph x with intra-run + * order preserved, or plain stream order when the line is RTL-placed. */ -function groupByBaseline(chars: ExtractedChar[], tolerance: number): ExtractedChar[][] { - const groups: ExtractedChar[][] = []; +function orderRuns(runs: TextRun[]): { chars: ExtractedChar[]; rtlPlaced: boolean } { + const streamOrder = runs.flatMap(run => run.chars); - for (const char of chars) { - // Find an existing group with a similar baseline - let foundGroup = false; + if (isRtlPlaced(streamOrder)) { + return { chars: streamOrder, rtlPlaced: true }; + } + + return { chars: mergeRunsByPosition(runs), rtlPlaced: false }; +} + +function mergeRunsByPosition(runs: TextRun[]): ExtractedChar[] { + const cursors = runs.map(run => ({ chars: run.chars, index: 0 })); + const merged: ExtractedChar[] = []; - for (const group of groups) { - const avgBaseline = calculateAverageBaseline(group); + for (;;) { + let next: (typeof cursors)[number] | null = null; - if (Math.abs(char.baseline - avgBaseline) <= tolerance) { - group.push(char); - foundGroup = true; - break; + for (const cursor of cursors) { + if (cursor.index >= cursor.chars.length) { + continue; + } + + if (!next || cursor.chars[cursor.index].bbox.x < next.chars[next.index].bbox.x) { + next = cursor; } } - if (!foundGroup) { - groups.push([char]); + if (!next) { + return merged; } + + merged.push(next.chars[next.index]); + next.index++; } +} - return groups; +/** + * Design tools (Figma, Canva) place LTR glyphs right-to-left via TJ + * adjustments, and genuine RTL text is emitted in logical order; in both + * cases stream order is reading order. Measured per character, not per run, + * so two runs drawn right-then-left (page number, then title) don't trigger + * it. Mixed bidi text needs a real bidi algorithm and is not handled. + */ +function isRtlPlaced(streamOrder: ExtractedChar[]): boolean { + if (streamOrder.length < 2) { + return false; + } + + let decreasingCount = 0; + + for (let i = 1; i < streamOrder.length; i++) { + if (streamOrder[i].bbox.x < streamOrder[i - 1].bbox.x) { + decreasingCount++; + } + } + + return decreasingCount / (streamOrder.length - 1) >= RTL_PLACED_THRESHOLD; } /** @@ -241,38 +423,33 @@ function groupIntoSpans( const prevChar = chars[i - 1]; const char = chars[i]; - // Check for font/size change const fontChanged = char.fontName !== currentFontName || Math.abs(char.fontSize - currentFontSize) > 0.5; - // Check for space gap — in RTL-placed lines, the "next" character in - // reading order sits to the left of the previous one, so the gap is - // measured from the left edge of prevChar to the right edge of char. + // In RTL-placed lines the next character sits to the left of the previous. + // A gap beside a real space glyph is the same logical space, not a second one. const gap = rtlPlaced ? prevChar.bbox.x - (char.bbox.x + char.bbox.width) : char.bbox.x - (prevChar.bbox.x + prevChar.bbox.width); const avgFontSize = (prevChar.fontSize + char.fontSize) / 2; - const needsSpace = gap > avgFontSize * spaceThreshold; + const needsSpace = + gap > avgFontSize * spaceThreshold && !isWhitespace(prevChar) && !isWhitespace(char); + + if (needsSpace) { + currentSpan.push(createSpaceChar(prevChar, char, rtlPlaced)); + } if (fontChanged) { - // Complete current span spans.push(buildSpan(currentSpan)); - // Start new span currentSpan = [char]; currentFontName = char.fontName; currentFontSize = char.fontSize; - } else if (needsSpace) { - // Add space to current span and continue - // We insert a synthetic space character - currentSpan.push(createSpaceChar(prevChar, char, rtlPlaced)); - currentSpan.push(char); } else { currentSpan.push(char); } } - // Complete final span if (currentSpan.length > 0) { spans.push(buildSpan(currentSpan)); } @@ -328,7 +505,7 @@ function createSpaceChar( /** * Calculate the average baseline of a group of characters. */ -function calculateAverageBaseline(chars: ExtractedChar[]): number { +function averageBaseline(chars: ExtractedChar[]): number { if (chars.length === 0) { return 0; } diff --git a/src/text/text-extractor.test.ts b/src/text/text-extractor.test.ts new file mode 100644 index 00000000..19aaf920 --- /dev/null +++ b/src/text/text-extractor.test.ts @@ -0,0 +1,295 @@ +import { PdfArray } from "#src/objects/pdf-array"; +import { PdfDict } from "#src/objects/pdf-dict"; +import { PdfName } from "#src/objects/pdf-name"; +import { PdfNumber } from "#src/objects/pdf-number"; +import type { PdfObject } from "#src/objects/pdf-object"; +import { PdfRef } from "#src/objects/pdf-ref"; +import { PdfStream } from "#src/objects/pdf-stream"; +import { describe, expect, it } from "vitest"; + +import { TextExtractor } from "./text-extractor"; +import { TextResources } from "./text-resources"; + +function helvetica(baseFont = "Helvetica"): PdfDict { + return PdfDict.of({ + Type: PdfName.of("Font"), + Subtype: PdfName.of("Type1"), + BaseFont: PdfName.of(baseFont), + }); +} + +function form( + content: string, + entries: { matrix?: number[]; resources?: PdfDict; bbox?: number[] } = {}, +): PdfStream { + const dict = PdfDict.of({ + Type: PdfName.of("XObject"), + Subtype: PdfName.of("Form"), + BBox: PdfArray.of(...(entries.bbox ?? [0, 0, 100, 100]).map(n => PdfNumber.of(n))), + }); + + if (entries.matrix) { + dict.set("Matrix", PdfArray.of(...entries.matrix.map(n => PdfNumber.of(n)))); + } + + if (entries.resources) { + dict.set("Resources", entries.resources); + } + + return new PdfStream(dict, new TextEncoder().encode(content)); +} + +function resourcesDict(fonts: Record, xobjects: Record = {}) { + return PdfDict.of({ + Font: PdfDict.of(fonts), + XObject: PdfDict.of(xobjects), + }); +} + +function resources(dict: PdfDict, table: Map = new Map()): TextResources { + return new TextResources(dict, ref => table.get(ref) ?? null); +} + +function extract(content: string, res: TextResources) { + return new TextExtractor(res).extract(new TextEncoder().encode(content)); +} + +function identityType0(): PdfDict { + return PdfDict.of({ + Type: PdfName.of("Font"), + Subtype: PdfName.of("Type0"), + BaseFont: PdfName.of("Test"), + Encoding: PdfName.of("Identity-H"), + DescendantFonts: PdfArray.of( + PdfDict.of({ + Type: PdfName.of("Font"), + Subtype: PdfName.of("CIDFontType2"), + BaseFont: PdfName.of("Test"), + DW: PdfNumber.of(500), + CIDSystemInfo: PdfDict.of({ + Registry: PdfName.of("Adobe"), + Ordering: PdfName.of("Identity"), + Supplement: PdfNumber.of(0), + }), + }), + ), + }); +} + +describe("TextExtractor", () => { + describe("word spacing", () => { + it("applies Tw to single-byte code 32", () => { + const res = resources(resourcesDict({ F1: helvetica() })); + + const chars = extract("BT /F1 10 Tf 100 Tw 0 0 Td (a b) Tj ET", res); + const b = chars.find(c => c.char === "b")!; + const space = chars.find(c => c.char === " ")!; + + expect(b.bbox.x).toBeCloseTo(space.bbox.x + space.bbox.width + 100); + }); + + it("does not apply Tw to byte 32 inside a two-byte code", () => { + const res = resources(resourcesDict({ F1: identityType0() })); + + const chars = extract("BT /F1 10 Tf 100 Tw 0 0 Td <00200041> Tj ET", res); + const a = chars.find(c => c.char === "A")!; + + expect(a.bbox.x).toBeCloseTo(5); + }); + + it("splits multi-byte strings by the font's CMap", () => { + const res = resources(resourcesDict({ F1: identityType0() })); + + const chars = extract("BT /F1 10 Tf 0 0 Td <004100420043> Tj ET", res); + + expect(chars.map(c => c.char).join("")).toBe("ABC"); + expect(chars.map(c => c.bbox.x)).toEqual([0, 5, 10]); + }); + }); + + describe("text rise", () => { + it("scales rise through the text matrix", () => { + const res = resources(resourcesDict({ F1: helvetica() })); + + const chars = extract("BT /F1 10 Tf 2 0 0 2 0 0 Tm 5 Ts (a) Tj ET", res); + + expect(chars[0].baseline).toBeCloseTo(10); + }); + + it("rotates rise with the text matrix", () => { + const res = resources(resourcesDict({ F1: helvetica() })); + + const chars = extract("BT /F1 10 Tf 0 1 -1 0 100 100 Tm 5 Ts (a) Tj ET", res); + + expect(chars[0].baseline).toBeCloseTo(100); + expect(chars[0].bbox.x + chars[0].bbox.width).toBeLessThan(100); + }); + }); + + describe("graphics state", () => { + it("restores font and size on Q", () => { + const res = resources( + resourcesDict({ F1: helvetica("Helvetica"), F2: helvetica("Helvetica-Bold") }), + ); + const chars = extract( + `BT /F1 10 Tf 0 0 Td (a) Tj ET + q BT /F2 20 Tf 0 0 Td (b) Tj ET Q + BT 0 0 Td (c) Tj ET`, + res, + ); + + expect(chars.map(c => [c.char, c.fontName, c.fontSize])).toEqual([ + ["a", "Helvetica", 10], + ["b", "Helvetica-Bold", 20], + ["c", "Helvetica", 10], + ]); + }); + }); + + describe("form XObjects", () => { + it("extracts text drawn inside a form", () => { + const inner = form("BT /F1 10 Tf 5 5 Td (in) Tj ET", { + resources: resourcesDict({ F1: helvetica() }), + }); + const res = resources(resourcesDict({ F1: helvetica() }, { Fm1: inner })); + + const chars = extract("BT /F1 10 Tf 0 0 Td (out) Tj ET /Fm1 Do", res); + + expect(chars.map(c => c.char).join("")).toBe("outin"); + }); + + it("positions form text through cm and the form Matrix", () => { + const inner = form("BT /F1 10 Tf 5 5 Td (x) Tj ET", { + matrix: [1, 0, 0, 1, 100, 0], + resources: resourcesDict({ F1: helvetica() }), + }); + const res = resources(resourcesDict({}, { Fm1: inner })); + + const chars = extract("q 1 0 0 1 0 200 cm /Fm1 Do Q", res); + + expect(chars).toHaveLength(1); + expect(chars[0].bbox.x).toBeCloseTo(105); + expect(chars[0].baseline).toBeCloseTo(205); + }); + + it("restores the CTM and font after the form", () => { + const inner = form("2 0 0 2 0 0 cm BT /F2 30 Tf 0 0 Td (x) Tj ET", { + resources: resourcesDict({ F2: helvetica() }), + }); + const res = resources(resourcesDict({ F1: helvetica() }, { Fm1: inner })); + + const chars = extract("BT /F1 10 Tf 0 0 Td (a) Tj ET /Fm1 Do BT 50 0 Td (b) Tj ET", res); + + const b = chars.find(c => c.char === "b")!; + + expect(b.fontSize).toBe(10); + expect(b.bbox.x).toBeCloseTo(50); + }); + + it("restores state even when the form leaves q/Q unbalanced", () => { + const inner = form("q q 2 0 0 2 0 0 cm BT /F1 10 Tf 0 0 Td (x) Tj ET", { + resources: resourcesDict({ F1: helvetica() }), + }); + const res = resources(resourcesDict({ F1: helvetica() }, { Fm1: inner })); + + const chars = extract("/Fm1 Do BT /F1 10 Tf 50 0 Td (b) Tj ET", res); + + expect(chars.find(c => c.char === "b")!.bbox.x).toBeCloseTo(50); + }); + + it("falls back to the invoking stream's resources when the form has none", () => { + const inner = form("BT /F1 10 Tf 0 0 Td (x) Tj ET"); + const res = resources(resourcesDict({ F1: helvetica() }, { Fm1: inner })); + + const chars = extract("/Fm1 Do", res); + + expect(chars.map(c => c.char).join("")).toBe("x"); + }); + + it("resolves forms and fonts through indirect references", () => { + const fontRef = PdfRef.of(10, 0); + const formRef = PdfRef.of(11, 0); + const table = new Map([ + [fontRef, helvetica()], + [ + formRef, + form("BT /F1 10 Tf 0 0 Td (x) Tj ET", { resources: resourcesDict({ F1: fontRef }) }), + ], + ]); + const res = resources(resourcesDict({}, { Fm1: formRef }), table); + + const chars = extract("/Fm1 Do", res); + + expect(chars.map(c => c.char).join("")).toBe("x"); + }); + + it("recurses into nested forms", () => { + const leaf = form("BT /F1 10 Tf 0 0 Td (leaf) Tj ET", { + resources: resourcesDict({ F1: helvetica() }), + }); + const branch = form("/Leaf Do", { resources: resourcesDict({}, { Leaf: leaf }) }); + const res = resources(resourcesDict({}, { Branch: branch })); + + const chars = extract("/Branch Do", res); + + expect(chars.map(c => c.char).join("")).toBe("leaf"); + }); + + it("terminates when a form invokes itself", () => { + const formRef = PdfRef.of(12, 0); + const table = new Map(); + const self = form("BT /F1 10 Tf 0 0 Td (x) Tj ET /Me Do", { + resources: resourcesDict({ F1: helvetica() }, { Me: formRef }), + }); + table.set(formRef, self); + const res = resources(resourcesDict({}, { Fm1: formRef }), table); + + const chars = extract("/Fm1 Do", res); + + expect(chars.map(c => c.char).join("")).toBe("x"); + }); + + it("ignores image XObjects and unknown names", () => { + const image = new PdfStream( + PdfDict.of({ Type: PdfName.of("XObject"), Subtype: PdfName.of("Image") }), + new Uint8Array([1, 2, 3]), + ); + const res = resources(resourcesDict({ F1: helvetica() }, { Im1: image })); + + const chars = extract("/Im1 Do /Missing Do BT /F1 10 Tf 0 0 Td (a) Tj ET", res); + + expect(chars.map(c => c.char).join("")).toBe("a"); + }); + }); +}); + +describe("TextResources", () => { + it("parses a font once when shared between page and form", () => { + const shared = helvetica(); + const fontRef = PdfRef.of(20, 0); + const table = new Map([[fontRef, shared]]); + const inner = form("", { resources: resourcesDict({ F9: fontRef }) }); + const res = resources(resourcesDict({ F1: fontRef }, { Fm1: inner }), table); + + const fromPage = res.getFont("F1"); + const fromForm = res.getForm("Fm1")!.resources.getFont("F9"); + + expect(fromPage).not.toBeNull(); + expect(fromForm).toBe(fromPage); + }); + + it("returns null for unknown or non-font entries", () => { + const res = resources(resourcesDict({ Bad: PdfNumber.of(1) })); + + expect(res.getFont("Bad")).toBeNull(); + expect(res.getFont("Nope")).toBeNull(); + expect(res.getForm("Nope")).toBeNull(); + }); + + it("handles a missing resources dictionary", () => { + const res = new TextResources(null, () => null); + + expect(res.getFont("F1")).toBeNull(); + expect(res.getForm("Fm1")).toBeNull(); + }); +}); diff --git a/src/text/text-extractor.ts b/src/text/text-extractor.ts index 4a9c41ff..aca5893b 100644 --- a/src/text/text-extractor.ts +++ b/src/text/text-extractor.ts @@ -11,56 +11,49 @@ import { type AnyOperation, type ContentToken, } from "#src/content/parsing/types"; -import type { PdfFont } from "#src/fonts/pdf-font"; +import type { PdfStream } from "#src/objects/pdf-stream"; +import type { TextResources } from "./text-resources"; import { TextState } from "./text-state"; import type { ExtractedChar } from "./types"; -/** - * Options for text extraction. - */ -export interface TextExtractorOptions { - /** - * Resolve a font name to a PdfFont object. - * Font names are keys in the /Resources/Font dictionary (e.g., "F1", "TT0"). - */ - resolveFont: (name: string) => PdfFont | null; -} +/** Cycles are caught by identity; this guards the call stack against deep chains. */ +const MAX_FORM_DEPTH = 32; /** * Extracts text from PDF content streams. */ export class TextExtractor { - private readonly resolveFont: (name: string) => PdfFont | null; - private readonly state: TextState; + private readonly state = new TextState(); private readonly chars: ExtractedChar[] = []; + private readonly activeForms = new Set(); - constructor(options: TextExtractorOptions) { - this.resolveFont = options.resolveFont; - this.state = new TextState(); - } + constructor(private readonly resources: TextResources) {} /** - * Extract all text from a content stream. + * Extract all text from a content stream and the form XObjects it invokes. * * @param contentBytes - The raw content stream bytes - * @returns Array of extracted characters with positions + * @returns Extracted characters with positions, in stream order */ extract(contentBytes: Uint8Array): ExtractedChar[] { - const parser = new ContentStreamParser(contentBytes); - const { operations } = parser.parse(); + this.run(contentBytes, this.resources); + + return this.chars; + } + + private run(contentBytes: Uint8Array, resources: TextResources): void { + const { operations } = new ContentStreamParser(contentBytes).parse(); for (const op of operations) { - this.processOperation(op); + this.processOperation(op, resources); } - - return this.chars; } /** * Process a single content stream operation. */ - private processOperation(op: AnyOperation): void { + private processOperation(op: AnyOperation, resources: TextResources): void { // Handle inline images separately if (isInlineImageOperation(op)) { return; // Skip inline images @@ -109,7 +102,7 @@ export class TextExtractor { break; case "Tf": - this.handleTf(operands); + this.handleTf(operands, resources); break; case "Tr": @@ -169,6 +162,38 @@ export class TextExtractor { this.state.moveToNextLine(); this.handleTj([operands[2]]); break; + + // XObjects + case "Do": + this.handleDo(operands, resources); + break; + } + } + + /** + * Interpret a form XObject as if wrapped in q/Q with its /Matrix applied + * (8.10.1). The state stack is restored to its prior depth even if the + * form's own q/Q are unbalanced. + */ + private handleDo(operands: ContentToken[], resources: TextResources): void { + const name = this.getName(operands[0]); + const form = name ? resources.getForm(name) : null; + + if (!form || this.activeForms.has(form.stream) || this.activeForms.size >= MAX_FORM_DEPTH) { + return; + } + + const depth = this.state.stackDepth; + + this.activeForms.add(form.stream); + this.state.saveGraphicsState(); + this.state.concatMatrix(...form.matrix.toArray()); + + try { + this.run(form.content, form.resources); + } finally { + this.state.restoreGraphicsStateTo(depth); + this.activeForms.delete(form.stream); } } @@ -189,13 +214,12 @@ export class TextExtractor { /** * Handle Tf (set font and size) operator. */ - private handleTf(operands: ContentToken[]): void { + private handleTf(operands: ContentToken[], resources: TextResources): void { const fontName = this.getName(operands[0]); const fontSize = this.getNumber(operands[1]); if (fontName) { - const font = this.resolveFont(fontName); - this.state.font = font; + this.state.font = resources.getFont(fontName); } this.state.fontSize = fontSize; @@ -245,78 +269,29 @@ export class TextExtractor { return; } - // Decode bytes to character codes based on font type - const codes = this.decodeStringToCodes(bytes, font); - - for (const code of codes) { - // Get Unicode character from font + for (const { code, length } of font.decode(bytes)) { + const width = font.getWidth(code); const char = font.toUnicode(code); - // Skip if we can't decode to Unicode - if (!char) { - // Still advance position even if we can't decode - const width = font.getWidth(code); - this.state.advanceChar(width, false); - continue; - } + // Single-byte code 32 only, never a byte 32 inside a multi-byte code (9.3.3) + const applyWordSpacing = length === 1 && code === 32; - // Get glyph width - const width = font.getWidth(code); + if (char) { + const bbox = this.state.getCharBbox(width); - // Calculate bounding box - const bbox = this.state.getCharBbox(width); - - // Create extracted character - this.chars.push({ - char, - bbox: { - x: bbox.x, - y: bbox.y, - width: bbox.width, - height: bbox.height, - }, - fontSize: this.state.effectiveFontSize, - fontName: font.baseFontName, - baseline: bbox.baseline, - sequenceIndex: this.chars.length, - }); - - // Advance text position - const isSpace = char === " " || char === "\u00A0"; // Space or non-breaking space - this.state.advanceChar(width, isSpace); - } - } - - /** - * Decode string bytes to character codes. - * - * For simple fonts (TrueType, Type1), each byte is a character code. - * For composite fonts (Type0/CID), bytes are decoded as 2-byte codes. - */ - private decodeStringToCodes(bytes: Uint8Array, font: PdfFont): number[] { - const codes: number[] = []; - - // Check if this is a composite font (Type0) - // Composite fonts use 2-byte character codes - if (font.subtype === "Type0") { - // Read as big-endian 16-bit values - for (let i = 0; i < bytes.length - 1; i += 2) { - const code = (bytes[i] << 8) | bytes[i + 1]; - codes.push(code); + this.chars.push({ + char, + bbox: { x: bbox.x, y: bbox.y, width: bbox.width, height: bbox.height }, + fontSize: this.state.effectiveFontSize, + fontName: font.baseFontName, + baseline: bbox.baseline, + sequenceIndex: this.chars.length, + }); } - // Handle odd byte at end (shouldn't happen in valid PDFs) - if (bytes.length % 2 === 1) { - codes.push(bytes[bytes.length - 1]); - } - } else { - // Simple font - each byte is a character code - for (const byte of bytes) { - codes.push(byte); - } + // Advance even for glyphs with no Unicode mapping + this.state.advanceChar(width, applyWordSpacing); } - - return codes; } /** diff --git a/src/text/text-resources.ts b/src/text/text-resources.ts new file mode 100644 index 00000000..fed156dd --- /dev/null +++ b/src/text/text-resources.ts @@ -0,0 +1,128 @@ +/** + * TextResources - Fonts and form XObjects reachable from a /Resources dict. + * + * Fonts are parsed on first use and cached by dictionary identity across a + * page and its forms. A form without its own /Resources inherits those of the + * invoking stream (deprecated by the spec, relied on by producers). + */ + +import { parseFont } from "#src/fonts/font-factory"; +import type { PdfFont } from "#src/fonts/pdf-font"; +import { parseToUnicode, type ToUnicodeMap } from "#src/fonts/to-unicode"; +import { Matrix } from "#src/helpers/matrix"; +import type { RefResolver } from "#src/helpers/types"; +import { PdfArray } from "#src/objects/pdf-array"; +import { PdfDict } from "#src/objects/pdf-dict"; +import { PdfNumber } from "#src/objects/pdf-number"; +import { PdfStream } from "#src/objects/pdf-stream"; + +export interface FormXObject { + /** Identity is used to detect recursion */ + stream: PdfStream; + content: Uint8Array; + matrix: Matrix; + resources: TextResources; +} + +export class TextResources { + private readonly forms = new Map(); + + constructor( + private readonly dict: PdfDict | null, + private readonly resolve: RefResolver, + private readonly fontCache: Map = new Map(), + ) {} + + getFont(name: string): PdfFont | null { + const entry = this.lookup("Font", name); + + if (!(entry instanceof PdfDict)) { + return null; + } + + let font = this.fontCache.get(entry); + + if (!font) { + font = parseFont(entry, { + resolver: this.resolve, + toUnicodeMap: this.parseToUnicode(entry), + }); + this.fontCache.set(entry, font); + } + + return font; + } + + /** Null for image XObjects and unknown names. */ + getForm(name: string): FormXObject | null { + let form = this.forms.get(name); + + if (form === undefined) { + form = this.loadForm(name); + this.forms.set(name, form); + } + + return form; + } + + private loadForm(name: string): FormXObject | null { + const stream = this.lookup("XObject", name); + + if (!(stream instanceof PdfStream)) { + return null; + } + + if (stream.getName("Subtype", this.resolve)?.value !== "Form") { + return null; + } + + const ownResources = stream.getDict("Resources", this.resolve); + + return { + stream, + content: stream.getDecodedData(), + matrix: readMatrix(stream.getArray("Matrix", this.resolve), this.resolve), + resources: ownResources + ? new TextResources(ownResources, this.resolve, this.fontCache) + : this, + }; + } + + private lookup(category: string, name: string) { + return this.dict?.getDict(category, this.resolve)?.get(name, this.resolve); + } + + private parseToUnicode(fontDict: PdfDict): ToUnicodeMap | null { + const stream = fontDict.get("ToUnicode", this.resolve); + + if (!(stream instanceof PdfStream)) { + return null; + } + + try { + return parseToUnicode(stream.getDecodedData()); + } catch { + return null; + } + } +} + +function readMatrix(array: PdfArray | undefined, resolve: RefResolver): Matrix { + if (!(array instanceof PdfArray) || array.length !== 6) { + return Matrix.identity(); + } + + const values: number[] = []; + + for (let i = 0; i < 6; i++) { + const item = array.at(i, resolve); + + if (!(item instanceof PdfNumber)) { + return Matrix.identity(); + } + + values.push(item.value); + } + + return Matrix.fromArray(values); +} diff --git a/src/text/text-state.ts b/src/text/text-state.ts index 82f49113..37c01fe3 100644 --- a/src/text/text-state.ts +++ b/src/text/text-state.ts @@ -20,9 +20,13 @@ export interface GraphicsState { } /** - * Text-specific state parameters. + * Text state parameters (9.3); all are part of the graphics state saved by q/Q. */ export interface TextStateParams { + /** Font (Tf) */ + font: PdfFont | null; + /** Font size (Tf) */ + fontSize: number; /** Character spacing (Tc) - extra space after each character */ charSpacing: number; /** Word spacing (Tw) - extra space after space characters */ @@ -77,33 +81,14 @@ export class TextState { /** Graphics state stack for q/Q operators */ private graphicsStateStack: GraphicsState[] = []; - /** - * Get the current position in user space. - * This applies the CTM to the text matrix position. - */ - get position(): { x: number; y: number } { - // Text position is at the origin of the text matrix - // transformed by the CTM - return this.ctm.transformPoint(this.tm.e, this.tm.f + this.rise); + /** Tm × CTM: maps scaled text space (after Tfs, Tz, Ts) to user space. */ + get textRenderingMatrix(): Matrix { + return this.tm.multiply(this.ctm); } - /** - * Get the effective font size accounting for text matrix and CTM scaling. - */ + /** Font size in user space. */ get effectiveFontSize(): number { - // The effective font size considers the text matrix scaling - // and the CTM scaling - const tmScale = this.tm.getScaleY(); - const ctmScale = this.ctm.getScaleY(); - - return Math.abs(this.fontSize * tmScale * ctmScale); - } - - /** - * Get the horizontal scale factor in user space. - */ - get effectiveHorizontalScale(): number { - return (this.horizontalScale / 100) * this.tm.getScaleX() * this.ctm.getScaleX(); + return Math.abs(this.fontSize * this.textRenderingMatrix.getScaleY()); } /** @@ -200,6 +185,8 @@ export class TextState { this.graphicsStateStack.push({ ctm: this.ctm.clone(), textState: { + font: this.font, + fontSize: this.fontSize, charSpacing: this.charSpacing, wordSpacing: this.wordSpacing, horizontalScale: this.horizontalScale, @@ -218,6 +205,8 @@ export class TextState { if (saved) { this.ctm = saved.ctm; + this.font = saved.textState.font; + this.fontSize = saved.textState.fontSize; this.charSpacing = saved.textState.charSpacing; this.wordSpacing = saved.textState.wordSpacing; this.horizontalScale = saved.textState.horizontalScale; @@ -227,6 +216,20 @@ export class TextState { } } + /** + * Number of saved graphics states. + */ + get stackDepth(): number { + return this.graphicsStateStack.length; + } + + /** Pop saved states until the stack is back at `depth`. */ + restoreGraphicsStateTo(depth: number): void { + while (this.graphicsStateStack.length > depth) { + this.restoreGraphicsState(); + } + } + /** * Modify current transformation matrix (cm operator). * Prepends the given matrix to the CTM. @@ -237,10 +240,11 @@ export class TextState { } /** - * Calculate the bounding box for a character at the current position. + * Bounding box for a glyph at the current position: laid out in scaled + * text space, rise included, and mapped through Tm × CTM. * - * @param width - Character width in glyph units (1000 = 1 em) - * @returns Bounding box in user space + * @param width - Glyph width in glyph units (1000 = 1 em) + * @returns Axis-aligned bounding box and baseline in user space */ getCharBbox(width: number): { x: number; @@ -249,106 +253,45 @@ export class TextState { height: number; baseline: number; } { - // Get font metrics - fall back to FontBBox if ascent/descent are 0 or missing - let ascender = this.font?.descriptor?.ascent; - let descender = this.font?.descriptor?.descent; - - // If ascent/descent are 0 or missing, try to derive from FontBBox - // This is common for Type3 fonts and some poorly-formed PDFs - if (!ascender && !descender && this.font?.descriptor?.fontBBox) { - const bbox = this.font.descriptor.fontBBox; - // FontBBox is [llx, lly, urx, ury] - ascent is ury, descent is lly - ascender = bbox[3]; // ury - descender = bbox[1]; // lly (typically negative or 0) - } - - // Final fallback to reasonable defaults - if (!ascender) { - ascender = 800; - } - if (descender === undefined || descender === null) { - descender = -200; - } - - // Calculate glyph dimensions in scaled text space (after fontSize) - // These are in "text rendering space" before Tm rotation/scaling - const glyphWidthScaled = (width / 1000) * this.fontSize * (this.horizontalScale / 100); - const glyphHeightScaled = ((ascender - descender) / 1000) * this.fontSize; - const descenderScaled = (descender / 1000) * this.fontSize; - - // Current text position from Tm (translation component) - const textX = this.tm.e; - const textY = this.tm.f + this.rise; - - // Build the combined transformation matrix: CTM * Tm - // This transforms from text rendering space to user space - const _combined = this.ctm.multiply(this.tm); + const { ascent, descent } = this.fontMetrics(); + const trm = this.textRenderingMatrix; - // Transform baseline point to user space - const baselinePoint = this.ctm.transformPoint(textX, textY); + const glyphWidth = (width / 1000) * this.fontSize * (this.horizontalScale / 100); + const bottom = (descent / 1000) * this.fontSize + this.rise; + const top = (ascent / 1000) * this.fontSize + this.rise; - // Define glyph corners in text rendering space (relative to origin) - // We'll compute positions relative to (0,0) then add the translation - // Bottom-left of glyph is at (0, descender), top-right is at (width, ascender) const corners = [ - { x: 0, y: descenderScaled }, // bottom-left - { x: glyphWidthScaled, y: descenderScaled }, // bottom-right - { x: glyphWidthScaled, y: descenderScaled + glyphHeightScaled }, // top-right - { x: 0, y: descenderScaled + glyphHeightScaled }, // top-left + trm.transformPoint(0, bottom), + trm.transformPoint(glyphWidth, bottom), + trm.transformPoint(glyphWidth, top), + trm.transformPoint(0, top), ]; - // Transform each corner through combined matrix (without translation, then add it) - // The Tm translation (e, f) positions the glyph, but we already have the baseline point - // We need to transform the glyph shape through the rotation/scale part of the matrix - const transformedCorners = corners.map(corner => { - // Transform the corner offset through Tm (rotation/scale only, no translation) - // Then through CTM - const tmRotated = { - x: this.tm.a * corner.x + this.tm.c * corner.y, - y: this.tm.b * corner.x + this.tm.d * corner.y, - }; - // Now transform through CTM (rotation/scale only) - return { - x: baselinePoint.x + (this.ctm.a * tmRotated.x + this.ctm.c * tmRotated.y), - y: baselinePoint.y + (this.ctm.b * tmRotated.x + this.ctm.d * tmRotated.y), - }; - }); - - // Compute axis-aligned bounding box from transformed corners - const xs = transformedCorners.map(c => c.x); - const ys = transformedCorners.map(c => c.y); - + const xs = corners.map(c => c.x); + const ys = corners.map(c => c.y); const minX = Math.min(...xs); - const maxX = Math.max(...xs); const minY = Math.min(...ys); - const maxY = Math.max(...ys); return { x: minX, y: minY, - width: maxX - minX, - height: maxY - minY, - baseline: baselinePoint.y, + width: Math.max(...xs) - minX, + height: Math.max(...ys) - minY, + baseline: trm.transformPoint(0, this.rise).y, }; } - /** - * Clone the current text state. - */ - clone(): TextState { - const copy = new TextState(); - copy.ctm = this.ctm.clone(); - copy.tm = this.tm.clone(); - copy.tlm = this.tlm.clone(); - copy.font = this.font; - copy.fontSize = this.fontSize; - copy.charSpacing = this.charSpacing; - copy.wordSpacing = this.wordSpacing; - copy.horizontalScale = this.horizontalScale; - copy.leading = this.leading; - copy.rise = this.rise; - copy.renderMode = this.renderMode; - - return copy; + /** Ascent/descent in glyph units, falling back to FontBBox (Type3) then defaults. */ + private fontMetrics(): { ascent: number; descent: number } { + const descriptor = this.font?.descriptor; + let ascent = descriptor?.ascent; + let descent = descriptor?.descent; + + if (!ascent && !descent && descriptor?.fontBBox) { + ascent = descriptor.fontBBox[3]; + descent = descriptor.fontBBox[1]; + } + + return { ascent: ascent || 800, descent: descent ?? -200 }; } } From f3bec67fc0d830cd85470d3f729d94fec7d11dcc Mon Sep 17 00:00:00 2001 From: Lucas Smith Date: Fri, 25 Sep 2026 13:15:16 +1000 Subject: [PATCH 2/3] fix(parser): recover from malformed content streams instead of failing A stray ")" or "}" made TokenReader loop forever on a zero-length keyword; an operator inside an unterminated array or dict, or a non-name dict key, threw and lost the whole page. The reader now always advances, the tokenizer ends the array/dict and re-emits the operator, and text extraction treats an undecodable or crashing form XObject as absent rather than dropping the page's own text. --- src/content/parsing/content-tokenizer.test.ts | 52 +++++++++++++++++++ src/content/parsing/content-tokenizer.ts | 26 ++++++++-- src/parser/token-reader.test.ts | 11 ++++ src/parser/token-reader.ts | 6 +++ src/text/text-extractor.test.ts | 18 +++++++ src/text/text-extractor.ts | 2 + src/text/text-resources.ts | 11 +++- 7 files changed, 121 insertions(+), 5 deletions(-) diff --git a/src/content/parsing/content-tokenizer.test.ts b/src/content/parsing/content-tokenizer.test.ts index 58adf771..465a4108 100644 --- a/src/content/parsing/content-tokenizer.test.ts +++ b/src/content/parsing/content-tokenizer.test.ts @@ -232,6 +232,58 @@ describe("ContentTokenizer", () => { expect(tokenizer.nextToken()).toEqual({ type: "number", value: 42 }); expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "m" }); }); + + it("ends an array at an operator and re-emits the operator", () => { + const tokenizer = new ContentTokenizer(encode("[(a) 5 TJ (b) Tj")); + + expect(tokenizer.nextToken()).toEqual({ + type: "array", + items: [ + { type: "string", value: encode("a"), hex: false }, + { type: "number", value: 5 }, + ], + }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "TJ" }); + expect(tokenizer.nextToken()).toMatchObject({ type: "string" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "Tj" }); + }); + + it("keeps the re-emitted operator in order when peeked", () => { + const tokenizer = new ContentTokenizer(encode("[(a) TJ q")); + + expect(tokenizer.peek()).toMatchObject({ type: "array" }); + expect(tokenizer.nextToken()).toMatchObject({ type: "array" }); + expect(tokenizer.peek()).toEqual({ type: "operator", name: "TJ" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "TJ" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "q" }); + }); + + it("skips a dict entry whose key is not a name", () => { + const tokenizer = new ContentTokenizer(encode("<< 1 2 /K 3 >> BDC")); + + expect(tokenizer.nextToken()).toEqual({ + type: "dict", + entries: new Map([["K", { type: "number", value: 3 }]]), + }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "BDC" }); + }); + + it("ends a dict at an operator and re-emits the operator", () => { + const tokenizer = new ContentTokenizer(encode("<< /K 1 /V BDC q")); + + expect(tokenizer.nextToken()).toMatchObject({ type: "dict" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "BDC" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "q" }); + }); + + it("stray ) or } does not stall the tokenizer", () => { + const tokenizer = new ContentTokenizer(encode("q ) } Q")); + + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "q" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: ")" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "}" }); + expect(tokenizer.nextToken()).toEqual({ type: "operator", name: "Q" }); + }); }); describe("peek and eof", () => { diff --git a/src/content/parsing/content-tokenizer.ts b/src/content/parsing/content-tokenizer.ts index b7f8fdb7..cc5550a7 100644 --- a/src/content/parsing/content-tokenizer.ts +++ b/src/content/parsing/content-tokenizer.ts @@ -25,6 +25,9 @@ export class ContentTokenizer { private peeked: ContentOrOperatorToken | null = null; + /** An operator found inside an unterminated array or dict, emitted next */ + private pushedBack: OperatorToken | null = null; + constructor(bytes: Uint8Array) { this.scanner = new Scanner(bytes); this.reader = new TokenReader(this.scanner); @@ -90,6 +93,13 @@ export class ContentTokenizer { } private readNext(): ContentOrOperatorToken | null { + if (this.pushedBack !== null) { + const token = this.pushedBack; + this.pushedBack = null; + + return token; + } + while (true) { const token = this.reader.nextToken(); @@ -188,8 +198,9 @@ export class ContentTokenizer { } if (item.type === "operator") { - // Operators shouldn't appear inside arrays - treat as error - throw new Error(`Unexpected operator in array: ${item.name}`); + // The closing ] was lost; end the array and let the operator run + this.pushedBack = item; + break; } items.push(item); @@ -220,8 +231,14 @@ export class ContentTokenizer { break; } + if (key.type === "operator") { + // The closing >> was lost; end the dict and let the operator run + this.pushedBack = key; + break; + } + if (key.type !== "name") { - throw new Error(`Expected name as dict key, got ${key.type}`); + continue; } // Read value @@ -232,7 +249,8 @@ export class ContentTokenizer { } if (value.type === "operator") { - throw new Error(`Unexpected operator as dict value: ${value.name}`); + this.pushedBack = value; + break; } entries.set(key.value, value); diff --git a/src/parser/token-reader.test.ts b/src/parser/token-reader.test.ts index 1ddf0833..72b39bd1 100644 --- a/src/parser/token-reader.test.ts +++ b/src/parser/token-reader.test.ts @@ -818,6 +818,17 @@ describe("TokenReader", () => { }); describe("delimiter parsing", () => { + it.each([")", "{", "}"])("always advances past a stray %s", stray => { + const r = reader(`a ${stray} b`); + const values: unknown[] = []; + + for (let t = r.nextToken(); t.type !== "eof"; t = r.nextToken()) { + values.push(t.type === "keyword" ? t.value : t.type); + } + + expect(values).toEqual(["a", stray, "b"]); + }); + it("parses array start [", () => { const r = reader("["); const token = r.nextToken(); diff --git a/src/parser/token-reader.ts b/src/parser/token-reader.ts index b5069984..9f80920a 100644 --- a/src/parser/token-reader.ts +++ b/src/parser/token-reader.ts @@ -600,6 +600,12 @@ export class TokenReader { this.scanner.advance(); } + // A stray delimiter such as ")" or "}" is not a regular char; consume it + // as a one-byte keyword so the reader always makes progress + if (this.scanner.position === start) { + this.scanner.advance(); + } + const value = this.extractText(start, this.scanner.position); return { type: "keyword", value, position }; diff --git a/src/text/text-extractor.test.ts b/src/text/text-extractor.test.ts index 19aaf920..8b4eab61 100644 --- a/src/text/text-extractor.test.ts +++ b/src/text/text-extractor.test.ts @@ -249,6 +249,24 @@ describe("TextExtractor", () => { expect(chars.map(c => c.char).join("")).toBe("x"); }); + it("keeps the page's text when a form cannot be decoded", () => { + const broken = new PdfStream( + PdfDict.of({ + Type: PdfName.of("XObject"), + Subtype: PdfName.of("Form"), + BBox: PdfArray.of(PdfNumber.of(0), PdfNumber.of(0), PdfNumber.of(1), PdfNumber.of(1)), + Filter: PdfName.of("LZWDecode"), + }), + new Uint8Array([0xff, 0xff, 0xff, 0xff]), + ); + const res = resources(resourcesDict({ F1: helvetica() }, { Fm1: broken })); + + const chars = extract("BT /F1 10 Tf 0 0 Td (a) Tj ET /Fm1 Do BT 20 0 Td (b) Tj ET", res); + + expect(chars.map(c => c.char).join("")).toBe("ab"); + expect(chars[1].bbox.x).toBeCloseTo(20); + }); + it("ignores image XObjects and unknown names", () => { const image = new PdfStream( PdfDict.of({ Type: PdfName.of("XObject"), Subtype: PdfName.of("Image") }), diff --git a/src/text/text-extractor.ts b/src/text/text-extractor.ts index aca5893b..1b9defd7 100644 --- a/src/text/text-extractor.ts +++ b/src/text/text-extractor.ts @@ -191,6 +191,8 @@ export class TextExtractor { try { this.run(form.content, form.resources); + } catch { + // A broken form must not take the page's own text with it } finally { this.state.restoreGraphicsStateTo(depth); this.activeForms.delete(form.stream); diff --git a/src/text/text-resources.ts b/src/text/text-resources.ts index fed156dd..245b6e1f 100644 --- a/src/text/text-resources.ts +++ b/src/text/text-resources.ts @@ -76,11 +76,20 @@ export class TextResources { return null; } + let content: Uint8Array; + + try { + content = stream.getDecodedData(); + } catch { + // Undecodable form: treat as absent rather than losing the page + return null; + } + const ownResources = stream.getDict("Resources", this.resolve); return { stream, - content: stream.getDecodedData(), + content, matrix: readMatrix(stream.getArray("Matrix", this.resolve), this.resolve), resources: ownResources ? new TextResources(ownResources, this.resolve, this.fontCache) From 8ab4928d3ddaf0f1b7872e1899ed0c6c02fdd649 Mon Sep 17 00:00:00 2001 From: Lucas Smith Date: Fri, 25 Sep 2026 15:28:24 +1000 Subject: [PATCH 3/3] fix: use fontbox CMap parser for CMap and ToUnicode --- src/document/forms/form-font.test.ts | 2 +- src/fontbox/cmap/parser.test.ts | 21 + src/fontbox/cmap/parser.ts | 10 +- src/fonts/cid-font.test.ts | 2 +- src/fonts/cid-font.ts | 30 +- src/fonts/cmap.test.ts | 243 +++++++----- src/fonts/cmap.ts | 390 +++---------------- src/fonts/composite-font.test.ts | 17 +- src/fonts/to-unicode.test.ts | 37 +- src/fonts/to-unicode.ts | 264 ++----------- src/integration/text/text-extraction.test.ts | 18 + 11 files changed, 315 insertions(+), 719 deletions(-) diff --git a/src/document/forms/form-font.test.ts b/src/document/forms/form-font.test.ts index 7fac6aa0..9589b86b 100644 --- a/src/document/forms/form-font.test.ts +++ b/src/document/forms/form-font.test.ts @@ -97,7 +97,7 @@ describe("ExistingFont", () => { baseFontName: "StubCID", cidToGidMap: new Uint16Array([0, 7]), embeddedProgram: new StubFontProgram(new Map([[0x0041, 7]]), new Set([7])), - toUnicodeMap: new ToUnicodeMap(new Map([[1, "A"]])), + toUnicodeMap: ToUnicodeMap.fromEntries([[1, "A"]]), }); const font = new ExistingFont("F0", null, null, true, cidFont); diff --git a/src/fontbox/cmap/parser.test.ts b/src/fontbox/cmap/parser.test.ts index 1488da85..8645aded 100644 --- a/src/fontbox/cmap/parser.test.ts +++ b/src/fontbox/cmap/parser.test.ts @@ -322,4 +322,25 @@ describe("CMap Parser", () => { expect(cmap.wmode).toBe(0); }); }); + + describe("malformed dictionaries", () => { + it("recovers from an unterminated << (PDFBox readDictionary semantics)", () => { + // From genko_oc_shiryo1.pdf: the CIDSystemInfo dict is never closed. + const cmap = parseCMap( + new TextEncoder().encode(`/CIDInit /ProcSet findresource begin 12 dict begin begincmap +CIDSystemInfo < <3A2A> endcodespacerange +2 beginbfchar +<0002> <0020> +<0519> <2461> +endbfchar +endcmap`), + ); + + expect(cmap.toUnicodeBytes(new Uint8Array([0x05, 0x19]))).toBe("\u2461"); + expect(cmap.toUnicodeBytes(new Uint8Array([0x00, 0x02]))).toBe(" "); + }); + }); }); diff --git a/src/fontbox/cmap/parser.ts b/src/fontbox/cmap/parser.ts index 9714d9d7..fc51752e 100644 --- a/src/fontbox/cmap/parser.ts +++ b/src/fontbox/cmap/parser.ts @@ -510,12 +510,10 @@ class CMapParser { const dict = new Map(); let key = this.parseNextToken(); - while (key !== null && !(isOperator(key) && key.op === ">>")) { - if (isName(key)) { - const value = this.parseNextToken(); - dict.set(key.name, value); - } - + // As in PDFBox, stop at the first non-name key so an unterminated + // dictionary doesn't swallow the rest of the CMap. + while (isName(key)) { + dict.set(key.name, this.parseNextToken()); key = this.parseNextToken(); } diff --git a/src/fonts/cid-font.test.ts b/src/fonts/cid-font.test.ts index 3c8b10c8..27782299 100644 --- a/src/fonts/cid-font.test.ts +++ b/src/fonts/cid-font.test.ts @@ -192,7 +192,7 @@ describe("CIDFont", () => { subtype: "CIDFontType2", baseFontName: "TestFont", cidToGidMap: new Uint16Array([0, 2]), - toUnicodeMap: new ToUnicodeMap(new Map([[1, "A"]])), + toUnicodeMap: ToUnicodeMap.fromEntries([[1, "A"]]), }); expect(font.tryGetCharCodeForUnicode(0x41)).toBe(1); diff --git a/src/fonts/cid-font.ts b/src/fonts/cid-font.ts index 02ade760..54945ac8 100644 --- a/src/fonts/cid-font.ts +++ b/src/fonts/cid-font.ts @@ -80,9 +80,6 @@ export class CIDFont { /** ToUnicode map from the parent Type0 font, if available */ private readonly toUnicodeMap: ToUnicodeMap | null; - /** Reverse ToUnicode map: Unicode code point -> character code */ - private unicodeToCharCodeMap: Map | null = null; - constructor(options: { subtype: CIDFontSubtype; baseFontName: string; @@ -239,32 +236,7 @@ export class CIDFont { } } - if (!this.toUnicodeMap) { - return null; - } - - if (!this.unicodeToCharCodeMap) { - this.unicodeToCharCodeMap = new Map(); - const unicodeToCharCodeMap = this.unicodeToCharCodeMap; - - this.toUnicodeMap.forEach((unicodeValue, charCode) => { - const chars = Array.from(unicodeValue); - - if (chars.length !== 1) { - return; - } - - const codePoint = chars[0].codePointAt(0); - - if (codePoint === undefined || unicodeToCharCodeMap.has(codePoint)) { - return; - } - - unicodeToCharCodeMap.set(codePoint, charCode); - }); - } - - return this.unicodeToCharCodeMap.get(unicode) ?? null; + return this.toUnicodeMap?.getCodeForUnicode(String.fromCodePoint(unicode)) ?? null; } private getCharCodeForGid(gid: number): number | null { diff --git a/src/fonts/cmap.test.ts b/src/fonts/cmap.test.ts index 315532ec..66606b0d 100644 --- a/src/fonts/cmap.test.ts +++ b/src/fonts/cmap.test.ts @@ -42,6 +42,12 @@ end return new TextEncoder().encode(cmap); } +const TWO_BYTE_CODESPACE = ` +1 begincodespacerange +<0000> +endcodespacerange +`; + describe("CMap", () => { describe("Identity-H", () => { it("should be identity mapping", () => { @@ -132,19 +138,20 @@ describe("CMap", () => { describe("lookup", () => { it("should return 0 for unmapped codes", () => { - const cmap = new CMap({ name: "Test" }); + const cmap = parseCMap(makeCMapStream(TWO_BYTE_CODESPACE)); expect(cmap.lookup(0x100)).toBe(0); }); it("should find direct char mappings", () => { - const cmap = new CMap({ - name: "Test", - charMappings: new Map([ - [0x0001, 100], - [0x0002, 200], - ]), - }); + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} +2 begincidchar +<0001> 100 +<0002> 200 +endcidchar +`), + ); expect(cmap.lookup(0x0001)).toBe(100); expect(cmap.lookup(0x0002)).toBe(200); @@ -152,10 +159,13 @@ describe("CMap", () => { }); it("should find range mappings", () => { - const cmap = new CMap({ - name: "Test", - rangeMappings: [{ start: 0x0100, end: 0x01ff, baseCID: 1000 }], - }); + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} +1 begincidrange +<0100> <01FF> 1000 +endcidrange +`), + ); expect(cmap.lookup(0x0100)).toBe(1000); expect(cmap.lookup(0x0101)).toBe(1001); @@ -164,38 +174,63 @@ describe("CMap", () => { }); it("should prefer direct mappings over ranges", () => { - const cmap = new CMap({ - name: "Test", - charMappings: new Map([[0x0105, 999]]), - rangeMappings: [{ start: 0x0100, end: 0x01ff, baseCID: 1000 }], - }); + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} +1 begincidchar +<0105> 999 +endcidchar +1 begincidrange +<0100> <01FF> 1000 +endcidrange +`), + ); expect(cmap.lookup(0x0105)).toBe(999); expect(cmap.lookup(0x0106)).toBe(1006); }); + + it("should distinguish codes by byte length when given", () => { + const cmap = parseCMap( + makeCMapStream(` +2 begincodespacerange +<00> <7F> +<8000> +endcodespacerange +2 begincidchar +<41> 1 +<0041> 2 +endcidchar +`), + ); + + expect(cmap.lookup(0x41, 1)).toBe(1); + expect(cmap.lookup(0x41, 2)).toBe(2); + }); }); describe("canEncode (non-identity)", () => { it("should return true for codes with direct mappings", () => { - const cmap = new CMap({ - name: "Test", - codespaceRanges: [{ low: 0x0000, high: 0xffff, numBytes: 2 }], - charMappings: new Map([ - [0x0041, 100], // 'A' - [0x0042, 101], // 'B' - ]), - }); + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} +2 begincidchar +<0041> 100 +<0042> 101 +endcidchar +`), + ); expect(cmap.canEncode("A")).toBe(true); expect(cmap.canEncode("AB")).toBe(true); }); it("should return true for codes within range mappings", () => { - const cmap = new CMap({ - name: "Test", - codespaceRanges: [{ low: 0x0000, high: 0xffff, numBytes: 2 }], - rangeMappings: [{ start: 0x0041, end: 0x005a, baseCID: 100 }], // A-Z - }); + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} +1 begincidrange +<0041> <005A> 100 +endcidrange +`), + ); expect(cmap.canEncode("A")).toBe(true); expect(cmap.canEncode("Z")).toBe(true); @@ -203,25 +238,65 @@ describe("CMap", () => { }); it("should return false for unmapped codes", () => { - const cmap = new CMap({ - name: "Test", - codespaceRanges: [{ low: 0x0000, high: 0xffff, numBytes: 2 }], - charMappings: new Map([[0x0041, 100]]), // Only 'A' - }); + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} +1 begincidchar +<0041> 100 +endcidchar +`), + ); expect(cmap.canEncode("B")).toBe(false); expect(cmap.canEncode("AB")).toBe(false); }); + }); + + describe("readCharCode (non-identity)", () => { + const shiftJis = parseCMap( + makeCMapStream(` +2 begincodespacerange +<20> <7E> +<8140> <9FFC> +endcodespacerange +`), + ); + + it("reads mixed-width codes by codespace range", () => { + const bytes = new Uint8Array([0x41, 0x81, 0x40, 0x20]); + + expect(shiftJis.readCharCode(bytes, 0)).toEqual({ code: 0x41, length: 1 }); + expect(shiftJis.readCharCode(bytes, 1)).toEqual({ code: 0x8140, length: 2 }); + expect(shiftJis.readCharCode(bytes, 3)).toEqual({ code: 0x20, length: 1 }); + }); - it("should return false for codes outside codespace", () => { - const cmap = new CMap({ - name: "Test", - codespaceRanges: [{ low: 0x0000, high: 0x00ff, numBytes: 1 }], - charMappings: new Map([[0x0041, 100]]), + it("matches codespace ranges byte-wise, not by integer value", () => { + const cmap = parseCMap( + makeCMapStream(` +2 begincodespacerange +<8140> <9FFC> +<810000> <9FFFFF> +endcodespacerange +`), + ); + + // 0x8200 is inside 0x8140..0x9ffc numerically but 0x00 < 0x40 + expect(cmap.readCharCode(new Uint8Array([0x82, 0x00, 0x00]), 0)).toEqual({ + code: 0x820000, + length: 3, + }); + expect(cmap.readCharCode(new Uint8Array([0x81, 0x5f, 0x00]), 0)).toEqual({ + code: 0x815f, + length: 2, }); + }); + + it("consumes the shortest code length on invalid codes", () => { + // 0x81 0x20: no 1-byte match, 0x20 < 0x40 so no 2-byte match + const bytes = new Uint8Array([0x81, 0x20, 0x41]); - // Code point 0x4E00 (Chinese character) is outside 0x00-0xFF codespace - expect(cmap.canEncode("\u4E00")).toBe(false); + expect(shiftJis.readCharCode(bytes, 0)).toEqual({ code: 0x81, length: 1 }); + expect(shiftJis.readCharCode(bytes, 1)).toEqual({ code: 0x20, length: 1 }); + expect(shiftJis.readCharCode(bytes, 2)).toEqual({ code: 0x41, length: 1 }); }); }); }); @@ -229,52 +304,43 @@ describe("CMap", () => { describe("parseCMap", () => { describe("codespace ranges", () => { it("should parse single codespace range", () => { - const data = makeCMapStream(` -1 begincodespacerange -<0000> -endcodespacerange -`); - - const cmap = parseCMap(data); - const ranges = cmap.getCodespaceRanges(); + const cmap = parseCMap(makeCMapStream(TWO_BYTE_CODESPACE)); - expect(ranges.length).toBe(1); - expect(ranges[0].low).toBe(0x0000); - expect(ranges[0].high).toBe(0xffff); - expect(ranges[0].numBytes).toBe(2); + expect(cmap.readCharCode(new Uint8Array([0x12, 0x34]), 0)).toEqual({ + code: 0x1234, + length: 2, + }); }); it("should parse multiple codespace ranges", () => { - const data = makeCMapStream(` + const cmap = parseCMap( + makeCMapStream(` 2 begincodespacerange <00> <7F> <8000> endcodespacerange -`); - - const cmap = parseCMap(data); - const ranges = cmap.getCodespaceRanges(); +`), + ); - expect(ranges.length).toBe(2); - expect(ranges[0].numBytes).toBe(1); - expect(ranges[1].numBytes).toBe(2); + expect(cmap.readCharCode(new Uint8Array([0x41]), 0)).toEqual({ code: 0x41, length: 1 }); + expect(cmap.readCharCode(new Uint8Array([0x80, 0x01]), 0)).toEqual({ + code: 0x8001, + length: 2, + }); }); }); describe("cidchar mappings", () => { it("should parse cidchar entries", () => { - const data = makeCMapStream(` -1 begincodespacerange -<0000> -endcodespacerange + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} 3 begincidchar <0001> 100 <0002> 200 <0003> 300 endcidchar -`); - - const cmap = parseCMap(data); +`), + ); expect(cmap.lookup(0x0001)).toBe(100); expect(cmap.lookup(0x0002)).toBe(200); @@ -284,16 +350,13 @@ endcidchar describe("cidrange mappings", () => { it("should parse cidrange entries", () => { - const data = makeCMapStream(` -1 begincodespacerange -<0000> -endcodespacerange + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} 1 begincidrange <0100> <01FF> 1000 endcidrange -`); - - const cmap = parseCMap(data); +`), + ); expect(cmap.lookup(0x0100)).toBe(1000); expect(cmap.lookup(0x0101)).toBe(1001); @@ -301,17 +364,14 @@ endcidrange }); it("should parse multiple cidrange entries", () => { - const data = makeCMapStream(` -1 begincodespacerange -<0000> -endcodespacerange + const cmap = parseCMap( + makeCMapStream(`${TWO_BYTE_CODESPACE} 2 begincidrange <0100> <01FF> 1000 <0200> <02FF> 2000 endcidrange -`); - - const cmap = parseCMap(data); +`), + ); expect(cmap.lookup(0x0100)).toBe(1000); expect(cmap.lookup(0x0200)).toBe(2000); @@ -320,24 +380,29 @@ endcidrange describe("CMap metadata", () => { it("should parse CMap name", () => { - const data = makeCMapStream(""); - const cmap = parseCMap(data); + const cmap = parseCMap(makeCMapStream("")); expect(cmap.name).toBe("TestCMap"); }); it("should use provided name as fallback", () => { - const data = new TextEncoder().encode("begincmap endcmap"); - const cmap = parseCMap(data, "FallbackName"); + const cmap = parseCMap(new TextEncoder().encode("begincmap endcmap"), "FallbackName"); expect(cmap.name).toBe("FallbackName"); }); it("should parse WMode for vertical", () => { - const data = makeCMapStream("/WMode 1 def"); - const cmap = parseCMap(data); + const cmap = parseCMap(makeCMapStream("/WMode 1 def")); expect(cmap.vertical).toBe(true); }); + + it("should recognize an embedded Identity-H by name", () => { + const cmap = parseCMap( + new TextEncoder().encode(`/CMapName /Identity-H def ${TWO_BYTE_CODESPACE}`), + ); + + expect(cmap.isIdentity).toBe(true); + }); }); }); diff --git a/src/fonts/cmap.ts b/src/fonts/cmap.ts index b0834ce8..d57db04b 100644 --- a/src/fonts/cmap.ts +++ b/src/fonts/cmap.ts @@ -1,234 +1,83 @@ /** - * CMap (Character Map) for Type0 fonts. + * CMap (character code to CID) for Type0 fonts. * - * CMaps define how character codes map to CIDs (Character IDs). - * They are used by Type0 (composite) fonts. - * - * Special cases: - * - Identity-H / Identity-V: Character codes map directly to CIDs - * - * Format (from PDF spec): - * - codespacerange: Defines valid code ranges - * - cidchar: Individual code to CID mappings - * - cidrange: Range mappings + * Thin wrapper over the fontbox CMap that adds the Identity-H/V predefined + * CMaps and the encode-side helpers CompositeFont needs. */ -/** - * Codespace range defines the valid range for character codes. - */ -export interface CodespaceRange { - /** Start of the range */ - low: number; - /** End of the range (inclusive) */ - high: number; - /** Number of bytes for codes in this range */ - numBytes: number; -} +import { + CMap as FontboxCMap, + CodespaceRange, + parseCMap as parseFontboxCMap, +} from "#src/fontbox/cmap"; + +const IDENTITY_NAMES = new Set(["Identity-H", "Identity-V"]); -/** - * CMap for mapping character codes to CIDs. - */ export class CMap { - /** CMap name (e.g., "Identity-H") */ readonly name: string; - - /** Whether this is a vertical writing mode CMap */ readonly vertical: boolean; - - /** Whether this is an identity CMap */ readonly isIdentity: boolean; - /** Codespace ranges */ - private readonly codespaceRanges: CodespaceRange[]; - - /** Direct character code to CID mappings */ - private readonly charMappings: Map; - - /** Range mappings: [start, end, baseCID] */ - private readonly rangeMappings: Array<{ - start: number; - end: number; - baseCID: number; - }>; - - constructor(options: { - name: string; - vertical?: boolean; - isIdentity?: boolean; - codespaceRanges?: CodespaceRange[]; - charMappings?: Map; - rangeMappings?: Array<{ start: number; end: number; baseCID: number }>; - }) { - this.name = options.name; - this.vertical = options.vertical ?? false; - this.isIdentity = options.isIdentity ?? false; - this.codespaceRanges = options.codespaceRanges ?? []; - this.charMappings = options.charMappings ?? new Map(); - this.rangeMappings = options.rangeMappings ?? []; - } - - /** - * Look up CID for a character code. - */ - lookup(code: number): number { - // For identity CMaps, code = CID - if (this.isIdentity) { - return code; - } - - // Check direct mappings first - const direct = this.charMappings.get(code); - - if (direct !== undefined) { - return direct; - } - - // Check range mappings - for (const range of this.rangeMappings) { - if (code >= range.start && code <= range.end) { - return range.baseCID + (code - range.start); - } - } - - // No mapping found - return 0 (.notdef) - return 0; - } - - /** - * Check if a character code is valid in this CMap's codespace. - */ - isValidCode(code: number): boolean { - for (const range of this.codespaceRanges) { - if (code >= range.low && code <= range.high) { - return true; - } - } + private readonly inner: FontboxCMap; - return this.codespaceRanges.length === 0; + constructor(inner: FontboxCMap, name?: string) { + this.inner = inner; + this.name = inner.name ?? name ?? ""; + this.vertical = inner.wmode === 1; + this.isIdentity = IDENTITY_NAMES.has(this.name); } /** - * Get the codespace ranges. + * Look up the CID for a character code. Returns 0 (.notdef) if unmapped. */ - getCodespaceRanges(): readonly CodespaceRange[] { - return this.codespaceRanges; + lookup(code: number, length?: number): number { + return length === undefined ? this.inner.toCID(code) : this.inner.toCIDWithLength(code, length); } /** * Encode text to character codes. - * For Identity-H encoding, codes are Unicode code points. + * Only Identity CMaps are supported, where codes are Unicode code points. */ encode(text: string): number[] { - if (this.isIdentity) { - // For Identity-H, each character becomes its code point - const codes: number[] = []; - - for (const char of text) { - const cp = char.codePointAt(0); - - if (cp !== undefined) { - codes.push(cp); - } - } - - return codes; + if (!this.isIdentity) { + throw new Error(`Encoding not supported for CMap: ${this.name}`); } - // For other CMaps, need to reverse lookup - // This is a simplified version - full CMap encoding is complex - throw new Error(`Encoding not supported for CMap: ${this.name}`); + return Array.from(text, char => char.codePointAt(0) ?? 0); } /** * Check if text can be encoded with this CMap. */ canEncode(text: string): boolean { - // Identity-H/V can encode any Unicode text. - // Characters outside BMP (> 0xFFFF) are represented as surrogate pairs - // in JavaScript strings, so each charCodeAt() returns a 16-bit value. if (this.isIdentity) { - // All UTF-16 code units are valid (0x0000-0xFFFF) return true; } - // For other CMaps, check if each UTF-16 code unit has a mapping for (let i = 0; i < text.length; i++) { - const code = text.charCodeAt(i); - - // Check if this code is in the codespace - if (!this.isValidCode(code)) { + if (this.inner.toCID(text.charCodeAt(i)) === 0) { return false; } - - // Check if there's a mapping for this code - if (!this.charMappings.has(code)) { - // Check range mappings - let found = false; - - for (const range of this.rangeMappings) { - if (code >= range.start && code <= range.end) { - found = true; - break; - } - } - - if (!found) { - return false; - } - } } return true; } /** - * Read a character code from a byte string. - * Returns the code and how many bytes were consumed. + * Read one character code from a byte string, following the CMap's + * codespace ranges. */ readCharCode(bytes: Uint8Array, offset: number): { code: number; length: number } { - // For identity, always 2 bytes - if (this.isIdentity) { - if (offset + 1 >= bytes.length) { - return { code: bytes[offset] ?? 0, length: 1 }; - } + const [code, length] = this.inner.readCode(bytes, offset); - const code = (bytes[offset] << 8) | bytes[offset + 1]; - - return { code, length: 2 }; - } - - // Try each codespace range to find matching length - let code = 0; - - for (let numBytes = 1; numBytes <= 4 && offset + numBytes <= bytes.length; numBytes++) { - // Build code from bytes - code = 0; - - for (let i = 0; i < numBytes; i++) { - code = (code << 8) | bytes[offset + i]; - } - - // Check if this matches a codespace range - for (const range of this.codespaceRanges) { - if (range.numBytes === numBytes && code >= range.low && code <= range.high) { - return { code, length: numBytes }; - } - } - } - - // Default to 1 byte - return { code: bytes[offset] ?? 0, length: 1 }; + return { code, length }; } /** * Get a predefined CMap by name. * - * Currently only Identity-H and Identity-V are supported. - * Other predefined CMaps (UniGB-UCS2-H, UniJIS-UCS2-H, etc.) return null. - * - * Note: Legacy CJK PDFs using predefined CMaps without ToUnicode maps - * may not extract text correctly. Modern PDFs include ToUnicode maps - * which take precedence for text extraction. + * Only Identity-H and Identity-V are available. The Adobe CJK CMaps + * (UniGB-UCS2-H, 90ms-RKSJ-H, ...) are not bundled and return null. */ static getPredefined(name: string): CMap | null { if (name === "Identity-H") { @@ -239,182 +88,37 @@ export class CMap { return CMap.identityV(); } - // Other predefined CMaps (UniGB-*, UniJIS-*, etc.) not implemented. - // Falls back to Identity-H in parseCompositeFont. return null; } - /** - * Create Identity-H CMap. - */ static identityH(): CMap { - return new CMap({ - name: "Identity-H", - vertical: false, - isIdentity: true, - codespaceRanges: [{ low: 0x0000, high: 0xffff, numBytes: 2 }], - }); + return identity("Identity-H", 0); } - /** - * Create Identity-V CMap. - */ static identityV(): CMap { - return new CMap({ - name: "Identity-V", - vertical: true, - isIdentity: true, - codespaceRanges: [{ low: 0x0000, high: 0xffff, numBytes: 2 }], - }); + return identity("Identity-V", 1); } } -/** - * Parse a CMap stream and return a CMap object. - * - * @param data - The raw CMap data (decoded stream content) - * @param name - Optional name for the CMap - * @returns Parsed CMap - */ -export function parseCMap(data: Uint8Array, name?: string): CMap { - const text = bytesToLatin1(data); - - const codespaceRanges: CodespaceRange[] = []; - const charMappings = new Map(); - const rangeMappings: Array<{ start: number; end: number; baseCID: number }> = []; - - let cmapName = name ?? ""; - let vertical = false; - - // Parse CMap name - const nameMatch = text.match(/\/CMapName\s+\/(\S+)/); - - if (nameMatch) { - cmapName = nameMatch[1]; - } - - // Parse WMode (vertical) - const wmodeMatch = text.match(/\/WMode\s+(\d)/); - - if (wmodeMatch) { - vertical = wmodeMatch[1] === "1"; - } - - // Check if this is an identity CMap - const isIdentity = cmapName === "Identity-H" || cmapName === "Identity-V"; - - // Parse codespace ranges - parseCodespaceRanges(text, codespaceRanges); +function identity(name: string, wmode: number): CMap { + const inner = new FontboxCMap(); + const low = new Uint8Array([0x00, 0x00]); + const high = new Uint8Array([0xff, 0xff]); - // Parse cidchar sections - parseCidCharSections(text, charMappings); + inner.name = name; + inner.wmode = wmode; + inner.addCodespaceRange(new CodespaceRange(low, high)); + inner.addCIDRange(low, high, 0); - // Parse cidrange sections - parseCidRangeSections(text, rangeMappings); - - return new CMap({ - name: cmapName, - vertical, - isIdentity, - codespaceRanges, - charMappings, - rangeMappings, - }); -} - -/** - * Parse all begincodespacerange/endcodespacerange sections. - */ -function parseCodespaceRanges(text: string, ranges: CodespaceRange[]): void { - const sectionRegex = /begincodespacerange\s*([\s\S]*?)\s*endcodespacerange/g; - - for (const match of text.matchAll(sectionRegex)) { - parseCodespaceContent(match[1], ranges); - } + return new CMap(inner); } /** - * Parse content within a codespacerange section. - * Format: - */ -function parseCodespaceContent(content: string, ranges: CodespaceRange[]): void { - const pairRegex = /<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>/g; - - for (const match of content.matchAll(pairRegex)) { - const low = parseInt(match[1], 16); - const high = parseInt(match[2], 16); - // Number of bytes is half the hex string length - const numBytes = Math.max(match[1].length, match[2].length) / 2; - ranges.push({ low, high, numBytes: Math.ceil(numBytes) }); - } -} - -/** - * Parse all begincidchar/endcidchar sections. - */ -function parseCidCharSections(text: string, mappings: Map): void { - const sectionRegex = /begincidchar\s*([\s\S]*?)\s*endcidchar/g; - - for (const match of text.matchAll(sectionRegex)) { - parseCidCharContent(match[1], mappings); - } -} - -/** - * Parse content within a cidchar section. - * Format: - */ -function parseCidCharContent(content: string, mappings: Map): void { - const pairRegex = /<([0-9A-Fa-f]+)>\s+(\d+)/g; - - for (const match of content.matchAll(pairRegex)) { - const code = parseInt(match[1], 16); - const cid = parseInt(match[2], 10); - mappings.set(code, cid); - } -} - -/** - * Parse all begincidrange/endcidrange sections. - */ -function parseCidRangeSections( - text: string, - ranges: Array<{ start: number; end: number; baseCID: number }>, -): void { - const sectionRegex = /begincidrange\s*([\s\S]*?)\s*endcidrange/g; - - for (const match of text.matchAll(sectionRegex)) { - parseCidRangeContent(match[1], ranges); - } -} - -/** - * Parse content within a cidrange section. - * Format: - */ -function parseCidRangeContent( - content: string, - ranges: Array<{ start: number; end: number; baseCID: number }>, -): void { - const rangeRegex = /<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s+(\d+)/g; - - for (const match of content.matchAll(rangeRegex)) { - const start = parseInt(match[1], 16); - const end = parseInt(match[2], 16); - const baseCID = parseInt(match[3], 10); - ranges.push({ start, end, baseCID }); - } -} - -/** - * Convert bytes to Latin-1 string (each byte becomes a character). + * Parse an embedded CMap stream. + * + * @param data - Decoded stream content + * @param name - Fallback name if the stream has no /CMapName */ -function bytesToLatin1(data: Uint8Array): string { - let result = ""; - - for (let i = 0; i < data.length; i++) { - result += String.fromCharCode(data[i]); - } - - return result; +export function parseCMap(data: Uint8Array, name?: string): CMap { + return new CMap(parseFontboxCMap(data), name); } diff --git a/src/fonts/composite-font.test.ts b/src/fonts/composite-font.test.ts index 909458af..ffc9ce4e 100644 --- a/src/fonts/composite-font.test.ts +++ b/src/fonts/composite-font.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { CIDFont, CIDWidthMap } from "./cid-font"; -import { CMap } from "./cmap"; +import { CMap, parseCMap } from "./cmap"; import { CompositeFont } from "./composite-font"; import { ToUnicodeMap } from "./to-unicode"; @@ -165,13 +165,14 @@ describe("CompositeFont", () => { it("follows the CMap codespace ranges for mixed-width codes", () => { // Shift-JIS style: single-byte ASCII, double-byte kanji - const cmap = new CMap({ - name: "Custom", - codespaceRanges: [ - { low: 0x20, high: 0x7e, numBytes: 1 }, - { low: 0x8140, high: 0x9ffc, numBytes: 2 }, - ], - }); + const cmap = parseCMap( + new TextEncoder().encode(` + 2 begincodespacerange + <20> <7e> + <8140> <9ffc> + endcodespacerange + `), + ); const font = new CompositeFont({ baseFontName: "TestFont", cmap, cidFont }); expect(font.decode(new Uint8Array([0x41, 0x81, 0x40, 0x20]))).toEqual([ diff --git a/src/fonts/to-unicode.test.ts b/src/fonts/to-unicode.test.ts index 7db8085a..1a131ad8 100644 --- a/src/fonts/to-unicode.test.ts +++ b/src/fonts/to-unicode.test.ts @@ -49,30 +49,35 @@ describe("ToUnicodeMap", () => { expect(map.has(0x42)).toBe(false); }); - it("should report size", () => { + it("should report emptiness", () => { const map = new ToUnicodeMap(); - expect(map.size).toBe(0); + expect(map.isEmpty).toBe(true); map.set(0x41, "A"); - expect(map.size).toBe(1); + expect(map.isEmpty).toBe(false); }); - it("should iterate with forEach", () => { - const map = new ToUnicodeMap(); - map.set(0x41, "A"); - map.set(0x42, "B"); + it("should build from entries", () => { + const map = ToUnicodeMap.fromEntries([ + [0x41, "A"], + [0x0100, "B"], + ]); - const entries: [string, number][] = []; - map.forEach((unicode, code) => { - entries.push([unicode, code]); - }); + expect(map.get(0x41)).toBe("A"); + expect(map.get(0x0100)).toBe("B"); + }); - expect(entries).toEqual([ - ["A", 0x41], - ["B", 0x42], + it("should reverse-lookup codes by Unicode", () => { + const map = ToUnicodeMap.fromEntries([ + [0x41, "A"], + [0x0100, "B"], ]); + + expect(map.getCodeForUnicode("A")).toBe(0x41); + expect(map.getCodeForUnicode("B")).toBe(0x0100); + expect(map.getCodeForUnicode("C")).toBeUndefined(); }); }); @@ -266,7 +271,7 @@ endbfrange const cmap = makeCMap(""); const map = parseToUnicode(cmap); - expect(map.size).toBe(0); + expect(map.isEmpty).toBe(true); }); it("should handle single character range", () => { @@ -279,7 +284,7 @@ endbfrange const map = parseToUnicode(cmap); expect(map.get(0x01)).toBe("A"); - expect(map.size).toBe(1); + expect(map.get(0x02)).toBeUndefined(); }); it("should handle lowercase hex", () => { diff --git a/src/fonts/to-unicode.ts b/src/fonts/to-unicode.ts index 39ad7241..fbbc23d7 100644 --- a/src/fonts/to-unicode.ts +++ b/src/fonts/to-unicode.ts @@ -1,265 +1,77 @@ /** - * ToUnicode CMap parser for text extraction. + * ToUnicode CMap for text extraction. * - * ToUnicode CMaps map character codes to Unicode strings, enabling - * text extraction from PDF content streams. - * - * Format (from PDF spec): - * - beginbfchar/endbfchar: Individual character mappings - * - * - * - beginbfrange/endbfrange: Range mappings - * - incrementing range - * [ ...] - array of destinations + * Thin wrapper over the fontbox CMap exposing the code-to-Unicode side. */ -/** - * ToUnicode map for converting character codes to Unicode strings. - */ +import { bytesToInt, CMap, intToBytes, parseCMap } from "#src/fontbox/cmap"; + export class ToUnicodeMap { - private readonly map: Map; + private readonly cmap: CMap; + + constructor(cmap: CMap = new CMap()) { + this.cmap = cmap; + } + + static fromEntries(entries: Iterable<[number, string]>): ToUnicodeMap { + const map = new ToUnicodeMap(); + + for (const [code, unicode] of entries) { + map.set(code, unicode); + } - constructor(map?: Map) { - this.map = map ?? new Map(); + return map; } /** * Get the Unicode string for a character code. */ get(code: number): string | undefined { - return this.map.get(code); + return this.cmap.toUnicode(code); } - /** - * Check if a character code has a mapping. - */ has(code: number): boolean { - return this.map.has(code); + return this.get(code) !== undefined; } - /** - * Set a mapping from character code to Unicode string. - */ - set(code: number, unicode: string): void { - this.map.set(code, unicode); + get isEmpty(): boolean { + return !this.cmap.hasUnicodeMappings(); } /** - * Get the number of mappings. + * Add a mapping. The code is stored at its minimal byte width. */ - get size(): number { - return this.map.size; + set(code: number, unicode: string): void { + this.cmap.addCharMapping(intToBytes(code, byteWidth(code)), unicode); } /** - * Iterate over all mappings. + * Reverse lookup: the character code that maps to a Unicode string. */ - forEach(callback: (unicode: string, code: number) => void): void { - this.map.forEach(callback); - } + getCodeForUnicode(unicode: string): number | undefined { + const bytes = this.cmap.getCodesFromUnicode(unicode); - /** - * Get the underlying map (for advanced use). - */ - getMap(): Map { - return this.map; + return bytes ? bytesToInt(bytes) : undefined; } } -/** - * Parse a ToUnicode CMap stream and return a mapping. - * - * @param data - The raw CMap data (decoded stream content) - * @returns Map from character codes to Unicode strings - */ -export function parseToUnicode(data: Uint8Array): ToUnicodeMap { - const map = new ToUnicodeMap(); - - // Decode as Latin-1 (each byte = one char, preserving binary data in hex strings) - const text = bytesToLatin1(data); - - // Parse beginbfchar sections - parseBfCharSections(text, map); - - // Parse beginbfrange sections - parseBfRangeSections(text, map); - - return map; -} - -/** - * Convert bytes to Latin-1 string (each byte becomes a character). - */ -function bytesToLatin1(data: Uint8Array): string { - // For CMap parsing, we just need to preserve the ASCII text and hex strings - // Each byte becomes a character with that code point - let result = ""; - for (let i = 0; i < data.length; i++) { - result += String.fromCharCode(data[i]); +function byteWidth(code: number): number { + if (code <= 0xff) { + return 1; } - return result; -} - -/** - * Parse all beginbfchar/endbfchar sections. - */ -function parseBfCharSections(text: string, map: ToUnicodeMap): void { - const sectionRegex = /beginbfchar\s*([\s\S]*?)\s*endbfchar/g; - for (const match of text.matchAll(sectionRegex)) { - parseBfCharContent(match[1], map); + if (code <= 0xffff) { + return 2; } -} - -/** - * Parse content within a bfchar section. - * Format: - */ -function parseBfCharContent(content: string, map: ToUnicodeMap): void { - // Match pairs of hex strings - const pairRegex = /<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>/g; - for (const match of content.matchAll(pairRegex)) { - const srcCode = parseInt(match[1], 16); - const dstUnicode = hexToUnicode(match[2]); - map.set(srcCode, dstUnicode); - } + return code <= 0xffffff ? 3 : 4; } /** - * Parse all beginbfrange/endbfrange sections. - */ -function parseBfRangeSections(text: string, map: ToUnicodeMap): void { - const sectionRegex = /beginbfrange\s*([\s\S]*?)\s*endbfrange/g; - - for (const match of text.matchAll(sectionRegex)) { - parseBfRangeContent(match[1], map); - } -} - -/** - * Parse content within a bfrange section. - * Format: - * - incrementing range - * [ ...] - array of destinations - */ -function parseBfRangeContent(content: string, map: ToUnicodeMap): void { - // Split into lines and process each - const lines = content.trim().split(/\n/); - - for (const line of lines) { - const trimmed = line.trim(); - - if (!trimmed) { - continue; - } - - // Try to match range with array: [ ...] - const arrayMatch = trimmed.match(/^<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*\[([\s\S]*)\]$/); - - if (arrayMatch) { - const lo = parseInt(arrayMatch[1], 16); - const hi = parseInt(arrayMatch[2], 16); - const arrayContent = arrayMatch[3]; - - // Extract all hex strings from the array - const hexMatches = arrayContent.match(/<([0-9A-Fa-f]+)>/g); - if (hexMatches) { - const destinations = hexMatches.map(h => hexToUnicode(h.slice(1, -1))); - - // Map each code in range to corresponding destination - for (let code = lo; code <= hi && code - lo < destinations.length; code++) { - map.set(code, destinations[code - lo]); - } - } - continue; - } - - // Try to match range with start code: - const rangeMatch = trimmed.match(/^<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>$/); - - if (rangeMatch) { - const lo = parseInt(rangeMatch[1], 16); - const hi = parseInt(rangeMatch[2], 16); - const dstHex = rangeMatch[3]; - - // Map the range with incrementing Unicode values - mapBfRange(lo, hi, dstHex, map); - } - } -} - -/** - * Map a range of character codes to incrementing Unicode values. - * - * The destination string is incremented for each code in the range. - * For multi-byte destinations, only the last byte is incremented, - * with carry to previous bytes when needed. - */ -function mapBfRange(lo: number, hi: number, dstHex: string, map: ToUnicodeMap): void { - // Validate range to prevent excessive iterations - const MAX_RANGE = 0xffffff; - if (hi - lo > MAX_RANGE) { - return; - } - - // Convert hex to array of code units (2 bytes per code unit for UTF-16BE) - const codeUnits = hexToCodeUnits(dstHex); - if (codeUnits.length === 0) { - return; - } - - for (let code = lo; code <= hi; code++) { - // Convert current code units to string - map.set(code, String.fromCharCode(...codeUnits)); - - // Increment the code units for next iteration - incrementCodeUnits(codeUnits); - } -} - -/** - * Convert hex string to array of 16-bit code units (UTF-16BE). - */ -function hexToCodeUnits(hex: string): number[] { - const codeUnits: number[] = []; - - // Pad to even length if needed - const padded = hex.length % 4 !== 0 ? hex.padStart(Math.ceil(hex.length / 4) * 4, "0") : hex; - - for (let i = 0; i < padded.length; i += 4) { - codeUnits.push(parseInt(padded.slice(i, i + 4), 16)); - } - - return codeUnits; -} - -/** - * Increment array of code units (with carry). - * Modifies the array in place. - */ -function incrementCodeUnits(codeUnits: number[]): void { - // Start from the last code unit and work backwards - for (let i = codeUnits.length - 1; i >= 0; i--) { - codeUnits[i]++; - if (codeUnits[i] <= 0xffff) { - // No overflow, done - break; - } - // Overflow - wrap and carry - codeUnits[i] = 0; - // Continue to increment previous code unit - } -} - -/** - * Convert hex string to Unicode string. + * Parse a ToUnicode CMap stream. * - * The hex string represents UTF-16BE encoded data: - * - 4 hex chars = 1 BMP character - * - 8 hex chars = surrogate pair (supplementary character) + * @param data - Decoded stream content */ -function hexToUnicode(hex: string): string { - const codeUnits = hexToCodeUnits(hex); - return String.fromCharCode(...codeUnits); +export function parseToUnicode(data: Uint8Array): ToUnicodeMap { + return new ToUnicodeMap(parseCMap(data)); } diff --git a/src/integration/text/text-extraction.test.ts b/src/integration/text/text-extraction.test.ts index 8c6a0b39..bd9645ce 100644 --- a/src/integration/text/text-extraction.test.ts +++ b/src/integration/text/text-extraction.test.ts @@ -260,6 +260,24 @@ describe("Text Extraction Integration", () => { expect(() => page!.extractText()).not.toThrow(); }); + + it("reads a ToUnicode CMap with an unterminated << dictionary", async () => { + const bytes = await loadFixture("malformed", "pdfbox/genko_oc_shiryo1.pdf"); + const pdf = await PDF.load(bytes); + const text = pdf.getPage(0)!.extractText().text; + + expect(text).toContain("統計法"); + expect(text).toContain("秘密の保護"); + }); + + it("reads a ToUnicode CMap written on a single line", async () => { + const bytes = await loadFixture("malformed", "pdfbox/PDFBOX-3208.pdf"); + const pdf = await PDF.load(bytes); + const text = pdf.getPage(5)!.extractText().text; + + expect(text).toContain("Tipo Documental"); + expect(text).toContain("Hipertensiva"); + }); }); describe("bounding boxes", () => {