diff --git a/docs/python.md b/docs/python.md index 242c23f34..2dd573491 100644 --- a/docs/python.md +++ b/docs/python.md @@ -113,6 +113,16 @@ headings = [ for item in pdf_inspector.extract_text_with_positions("tagged.pdf") if item.mcid is not None and roles.get((item.page, item.mcid), "").startswith("H") ] + +# Markdown plus typed layout blocks for citation grounding. Same Full-mode +# pipeline as process_pdf (no layout model); each block carries a normalized +# 0-1 page-space bbox (top-left origin) and an exact [start, end) byte span +# into the returned markdown: +result = pdf_inspector.extract_layout_blocks("document.pdf") +md = result.markdown.encode("utf-8") +for block in result.blocks: + start, end = block.markdown_span + print(block.block_type, block.page, md[start:end].decode("utf-8")) ``` ## API reference @@ -137,6 +147,8 @@ headings = [ | `extract_pages_markdown_bytes(data, pages=None)` | Per-page Markdown from bytes | | `extract_structure_elements(path, pages=None)` | Structure-tree elements from tagged PDFs (page, mcid, role) | | `extract_structure_elements_bytes(data, pages=None)` | Structure-tree elements from bytes | +| `extract_layout_blocks(path)` | Markdown + typed layout blocks with bbox and markdown spans | +| `extract_layout_blocks_bytes(data)` | Layout blocks from bytes | ## Types @@ -227,6 +239,23 @@ class StructureElement: # extract_structure_elements mcid: int role: str # "H1".."H6", "P", "Table", ... (resolved via /RoleMap) +class LayoutBlocksResult: # extract_layout_blocks + markdown: str # block spans are byte offsets into this string + blocks: list[LayoutBlock] # reading order, non-overlapping ascending spans + pdf_type: str + page_count: int + +class LayoutBlock: + block_type: str # "title" | "section_header" | "text" | "list_item" + # | "caption" | "code" | "table" | "picture" + label: str | None # "H2".."H6" for section_header blocks + page: int # 1-indexed + bbox: tuple[float, float, float, float] | None # normalized 0-1, top-left origin + markdown_span: tuple[int, int] # [start, end) byte offsets into markdown + source: str # always "native_text" + layout_confidence: float | None # always None (no layout model runs) + ocr_confidence: float | None # always None (native text, not OCR) + class RegionText: # extract_text_in_regions text: str needs_ocr: bool diff --git a/docs/rust-api.md b/docs/rust-api.md index 13ed38ac1..471b2ee4a 100644 --- a/docs/rust-api.md +++ b/docs/rust-api.md @@ -487,6 +487,36 @@ for item in extract_text_with_positions("tagged.pdf")? { } ``` +Extract Markdown plus typed layout blocks for citation grounding. This runs +the same Full-mode pipeline as `process_pdf` (no layout model) and records +each fragment the Markdown converter emits — headings, paragraphs, list +items, captions, code, tables, images — as a block with a normalized 0–1 +page-space bbox (top-left origin) and an exact `[start, end)` byte span into +the returned markdown: + +```rust +use pdf_inspector::{extract_layout_blocks, LayoutBlockType}; + +let result = extract_layout_blocks("document.pdf")?; +for block in &result.blocks { + let (start, end) = block.markdown_span; + let text = &result.markdown[start..end]; + // block.block_type: Title | SectionHeader | Text | ListItem | Caption + // | Code | Table | Picture + // block.label: Some("H2".."H6") for SectionHeader blocks + // block.bbox: Some([x0, y0, x1, y1]) normalized to the page box + // block.source: "native_text"; confidence fields stay None (no model) + println!("p{} {}: {}", block.page, block.block_type.as_str(), text); +} +``` + +Spans are ascending and non-overlapping, so slicing `result.markdown` with +them recovers the document in reading order. Unlike the default +`process_pdf` Markdown, the payload includes image placeholders so figures +surface as `picture` blocks. Scanned/image-based PDFs return the +classification with empty `markdown` and no blocks. The default +`process_pdf` output is unaffected by recording. + ## Processing modes | Mode | What it does | Returns | @@ -514,6 +544,8 @@ for item in extract_text_with_positions("tagged.pdf")? { | `extract_pages_markdown_mem(bytes, pages)` | Per-page Markdown from bytes | | `extract_structure_elements(path, pages)` | Structure-tree elements from tagged PDFs (page, mcid, role) | | `extract_structure_elements_mem(bytes, pages)` | Structure-tree elements from bytes | +| `extract_layout_blocks(path)` | Markdown + typed layout blocks with bbox and markdown spans | +| `extract_layout_blocks_mem(bytes)` | Layout blocks from bytes | Low-level detection functions are also available via the `detector` module (`detect_pdf_type`, `detect_pdf_type_with_config`, etc.) for callers who need `PdfTypeResult` instead of `PdfProcessResult`. @@ -531,6 +563,9 @@ Low-level detection functions are also available via the `detector` module (`det | `LayoutComplexity` | Layout analysis: is_complex, pages_with_tables, pages_with_columns | | `TextItem` | Text with position, font info, page number, and optional structure-tree `mcid` | | `StructureElement` | Tagged-PDF structure reference: page (1-indexed), mcid, role (`"H1"`..`"H6"`, `"P"`, …) | +| `LayoutBlocksResult` | Markdown plus typed layout blocks whose byte spans index into it | +| `LayoutBlock` | One block: type, label, page, normalized bbox, markdown_span, source, confidences | +| `LayoutBlockType` | `Title`, `SectionHeader`, `Text`, `ListItem`, `Caption`, `Code`, `Table`, `Picture` | | `MarkdownOptions` | Configuration for Markdown formatting (page numbers, etc.) | | `PageMarkdown` | Per-page result: page (0-indexed), markdown, needs_ocr | | `PagesExtractionResult` | Per-page output + 1-indexed pages_with_tables / pages_with_columns / pages_needing_ocr, is_complex | diff --git a/napi/README.md b/napi/README.md index bbaacb3bc..84b8d8b21 100644 --- a/napi/README.md +++ b/napi/README.md @@ -118,9 +118,27 @@ for (const region of result[0].regions) { } ``` +### `extractLayoutBlocks(buffer: Buffer): LayoutBlocksResult` + +Extract Markdown plus typed layout blocks for citation grounding. Runs the same Full-mode pipeline as `processPdf` (no layout model) and records each emitted fragment — headings, paragraphs, list items, captions, code, tables, images — as a block with a normalized 0–1 page-space bbox (top-left origin) and an exact `[start, end)` byte span into the returned markdown. Unlike the default `processPdf` markdown, this payload includes image placeholders so figures surface as `picture` blocks. + +Synchronous: like `extractText`, `extractStructureElements`, and `extractTextInRegions`, it parses on the calling thread and has no async variant. + +```typescript +import { extractLayoutBlocks } from '@firecrawl/pdf-inspector' + +const { markdown, blocks } = extractLayoutBlocks(pdf) +const bytes = Buffer.from(markdown, 'utf8') + +for (const block of blocks) { + const [start, end] = block.markdownSpan + console.log(block.blockType, block.page, bytes.subarray(start, end).toString('utf8')) +} +``` + ### Async variants -`processPdf`, `classifyPdf`, and `extractPagesMarkdown` are synchronous and parse on the calling thread — in Node, that's the event loop. For a one-off call in a script that's fine, but in a server a large document can hold the loop for tens to hundreds of milliseconds. +`processPdf`, `classifyPdf`, `extractPagesMarkdown`, and the other extraction functions (including `extractLayoutBlocks`) are synchronous and parse on the calling thread — in Node, that's the event loop. For a one-off call in a script that's fine, but in a server a large document can hold the loop for tens to hundreds of milliseconds. `processPdfAsync`, `classifyPdfAsync`, and `extractPagesMarkdownAsync` take the same arguments and produce the same results, but run the parse on the libuv thread pool and return a promise, keeping the event loop free. The input buffer is copied before the call returns, so it's safe to reuse or mutate immediately: @@ -160,6 +178,25 @@ interface RegionText { ocrReason?: string // "suspected_garbled_text" when known } +interface LayoutBlocksResult { + markdown: string // block spans are byte offsets into this string + blocks: LayoutBlock[] // reading order, non-overlapping ascending spans + pdfType: string + pageCount: number +} + +interface LayoutBlock { + blockType: string // "title" | "section_header" | "text" | "list_item" + // | "caption" | "code" | "table" | "picture" + label?: string // "H2".."H6" for section_header blocks + page: number // 1-indexed + bbox?: number[] // [x0, y0, x1, y1] normalized 0-1, top-left origin + markdownSpan: number[] // [start, end) byte offsets into markdown + source: string // always "native_text" + layoutConfidence?: number // always undefined (no layout model runs) + ocrConfidence?: number // always undefined (native text, not OCR) +} + interface OcrPdfResult { markdown: string pages: OcrPageResult[] // 1-indexed pages + provenance diff --git a/napi/src/lib.rs b/napi/src/lib.rs index 7801df38e..91ebbbc57 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -570,6 +570,88 @@ pub fn extract_structure_elements( }) } +/// One typed layout block from the Full-mode Markdown pipeline. +#[napi(object)] +pub struct LayoutBlockJs { + /// Hosted block type: "title", "section_header", "text", "list_item", + /// "caption", "code", "table", or "picture". + pub block_type: String, + /// Heading level label ("H2".."H6") for section_header blocks, `None` + /// otherwise. + pub label: Option, + /// 1-indexed page number the block was emitted from. + pub page: u32, + /// Normalized `[x0, y0, x1, y1]` in 0–1 page space with a top-left + /// origin, or `None` when no positioned geometry is available. + pub bbox: Option>, + /// `[start, end)` byte offsets into the result's `markdown`. + pub markdown_span: Vec, + /// Provenance of the block's text; always `"native_text"`. + pub source: String, + /// Layout-model confidence; always `None` (no layout model runs). + pub layout_confidence: Option, + /// OCR confidence; always `None` (the text is native, not OCR output). + pub ocr_confidence: Option, +} + +/// Markdown plus typed layout blocks whose spans index into it. +#[napi(object)] +pub struct LayoutBlocksResultJs { + /// Markdown assembled from the recorded fragments; block spans are + /// byte offsets into this string. + pub markdown: String, + /// Typed blocks in reading order with non-overlapping ascending spans. + pub blocks: Vec, + pub pdf_type: PdfType, + pub page_count: u32, +} + +/// Extract Markdown plus typed layout blocks for citation grounding. +/// +/// Runs the same Full-mode extract+convert pipeline as [`processPdf`] +/// (no layout model) and records each emitted fragment as a typed block +/// with a normalized 0–1 page-space bbox (top-left origin) and an exact +/// `[start, end)` byte span into the returned markdown. +/// +/// Unlike the default `processPdf` markdown, the payload includes image +/// placeholders so figures surface as `picture` blocks. +/// +/// For scanned/image-based PDFs the result carries the classification with +/// empty `markdown` and no blocks. +#[napi] +pub fn extract_layout_blocks(buffer: Buffer) -> Result { + let bytes: Vec = buffer.to_vec(); + catch_panic("extract_layout_blocks", move || { + let result = pdf_inspector::extract_layout_blocks_mem(&bytes) + .map_err(|e| to_napi_err(e, "extract_layout_blocks"))?; + Ok(LayoutBlocksResultJs { + markdown: result.markdown, + blocks: result + .blocks + .into_iter() + .map(|block| LayoutBlockJs { + block_type: block.block_type.as_str().to_string(), + label: block.label, + page: block.page, + bbox: block + .bbox + .map(|bbox| bbox.iter().map(|&v| v as f64).collect()), + markdown_span: markdown_span_to_js(block.markdown_span), + source: block.source, + layout_confidence: block.layout_confidence.map(f64::from), + ocr_confidence: block.ocr_confidence.map(f64::from), + }) + .collect(), + pdf_type: convert_pdf_type(result.pdf_type), + page_count: result.page_count, + }) + }) +} + +fn markdown_span_to_js((start, end): (usize, usize)) -> Vec { + vec![start as f64, end as f64] +} + /// Extract text within bounding-box regions from a PDF. /// /// For hybrid OCR: layout model detects regions in rendered images, @@ -1130,3 +1212,17 @@ pub fn extract_pages_markdown_async( pages, }) } + +#[cfg(all(test, target_pointer_width = "64"))] +mod tests { + use super::markdown_span_to_js; + + #[test] + fn markdown_span_does_not_wrap_at_u32_boundary() { + let start = u32::MAX as usize + 1; + assert_eq!( + markdown_span_to_js((start, start + 1)), + vec![4294967296.0, 4294967297.0] + ); + } +} diff --git a/napi/test.mjs b/napi/test.mjs index c1cd0ca1f..90a331c36 100644 --- a/napi/test.mjs +++ b/napi/test.mjs @@ -10,6 +10,7 @@ import { extractText, extractTextWithPositions, extractStructureElements, + extractLayoutBlocks, extractTextInRegions, detectVectorGridInRegion, extractPagesMarkdown, @@ -125,6 +126,54 @@ assert.ok(page1Elements.every(e => e.page === 1)); assert.deepEqual(extractStructureElements(fixture), []); console.log(' extractStructureElements: OK'); +// --- extractLayoutBlocks --- +console.log('Testing extractLayoutBlocks...'); +const layoutBlocks = extractLayoutBlocks(fixture); +assert.equal(layoutBlocks.pdfType, 'TextBased'); +assert.equal(layoutBlocks.pageCount, 3); +assert.ok(layoutBlocks.markdown.length > 0); +assert.ok(layoutBlocks.blocks.length > 0); +const allowedBlockTypes = new Set([ + 'title', + 'section_header', + 'text', + 'list_item', + 'caption', + 'code', + 'table', + 'picture', +]); +assert.ok(layoutBlocks.blocks.every(b => allowedBlockTypes.has(b.blockType))); +assert.ok(layoutBlocks.blocks.every(b => b.source === 'native_text')); +assert.ok(layoutBlocks.blocks.every(b => b.layoutConfidence === undefined)); +assert.ok(layoutBlocks.blocks.every(b => b.ocrConfidence === undefined)); + +// Spans are byte offsets into the payload's own markdown: ascending, +// non-overlapping, on UTF-8 character boundaries (fatal decoder throws on +// a mid-codepoint slice), and each slice is non-empty content. +const markdownBytes = Buffer.from(layoutBlocks.markdown, 'utf8'); +const fatalUtf8 = new TextDecoder('utf-8', { fatal: true }); +let prevSpanEnd = 0; +for (const block of layoutBlocks.blocks) { + const [start, end] = block.markdownSpan; + assert.ok(Number.isSafeInteger(start) && Number.isSafeInteger(end)); + assert.ok(start >= prevSpanEnd && start < end && end <= markdownBytes.length); + assert.ok(fatalUtf8.decode(markdownBytes.subarray(start, end)).trim().length > 0); + prevSpanEnd = end; +} + +// Bboxes are normalized 0-1 page space (top-left origin). +assert.ok(layoutBlocks.blocks.some(b => b.bbox !== undefined)); +for (const block of layoutBlocks.blocks) { + assert.ok(block.page >= 1 && block.page <= layoutBlocks.pageCount); + if (block.bbox === undefined) continue; + const [x0, y0, x1, y1] = block.bbox; + assert.ok(x0 >= 0 && x0 <= x1 && x1 <= 1); + assert.ok(y0 >= 0 && y0 <= y1 && y1 <= 1); +} +assert.ok(layoutBlocks.blocks.some(b => b.blockType === 'table')); +console.log(' extractLayoutBlocks: OK'); + // --- extractTextInRegions --- console.log('Testing extractTextInRegions...'); const regionResults = extractTextInRegions(fixture, [ diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index c80a4a9c3..68048d446 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -114,6 +114,39 @@ class StructureElement: role: str """Standard structure type name ("H1".."H6", "P", "Table", "TD", ...).""" +class LayoutBlock: + """One typed layout block from the Full-mode Markdown pipeline.""" + block_type: str + """Hosted block type: "title", "section_header", "text", "list_item", + "caption", "code", "table", or "picture".""" + label: Optional[str] + """Heading level label ("H2".."H6") for section_header blocks, None + otherwise.""" + page: int + """1-indexed page number the block was emitted from.""" + bbox: Optional[tuple[float, float, float, float]] + """Normalized (x0, y0, x1, y1) in 0-1 page space with a top-left origin, + or None when no positioned geometry is available.""" + markdown_span: tuple[int, int] + """[start, end) byte offsets into LayoutBlocksResult.markdown.""" + source: str + """Provenance of the block's text; always "native_text".""" + layout_confidence: Optional[float] + """Layout-model confidence; always None (no layout model runs).""" + ocr_confidence: Optional[float] + """OCR confidence; always None (the text is native, not OCR output).""" + +class LayoutBlocksResult: + """Markdown plus typed layout blocks whose spans index into it.""" + markdown: str + """Markdown assembled from the recorded fragments; block spans are byte + offsets into this string.""" + blocks: list[LayoutBlock] + """Typed blocks in reading order with non-overlapping ascending spans.""" + pdf_type: str + """'text_based', 'scanned', 'image_based', or 'mixed'.""" + page_count: int + class RegionText: """Extracted text for a single region.""" text: str @@ -248,6 +281,29 @@ def extract_structure_elements_bytes(data: bytes, pages: Optional[list[int]] = N """ ... +def extract_layout_blocks(path: str) -> LayoutBlocksResult: + """Extract Markdown plus typed layout blocks from a PDF file. + + Runs the same Full-mode extract+convert pipeline as :func:`process_pdf` + (no layout model) and records each emitted fragment as a typed block with + a normalized 0-1 page-space bbox (top-left origin) and an exact + [start, end) byte span into the returned markdown. + + Unlike the default process_pdf markdown, the payload includes image + placeholders so figures surface as "picture" blocks. + + For scanned/image-based PDFs the result carries the classification with + empty markdown and no blocks. + """ + ... + +def extract_layout_blocks_bytes(data: bytes) -> LayoutBlocksResult: + """Extract Markdown plus typed layout blocks from PDF bytes. + + See :func:`extract_layout_blocks` for details. + """ + ... + def extract_text_in_regions( path: str, page_regions: list[tuple[int, list[list[float]]]], diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index b2e07a5bb..b1499677a 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -258,6 +258,59 @@ fn extract_items_json( .map(|items| format_items_json(&items)) } +fn optional_confidence_json(value: Option) -> String { + value + .filter(|value| value.is_finite()) + .map(|value| format!("{value:.4}")) + .unwrap_or_else(|| "null".to_string()) +} + +fn format_layout_blocks_json(result: &pdf_inspector::LayoutBlocksResult) -> String { + let pdf_type = match result.pdf_type { + pdf_inspector::PdfType::TextBased => "text_based", + pdf_inspector::PdfType::Scanned => "scanned", + pdf_inspector::PdfType::ImageBased => "image_based", + pdf_inspector::PdfType::Mixed => "mixed", + }; + let blocks = result + .blocks + .iter() + .map(|block| { + let label = block + .label + .as_ref() + .map(|label| format!(r#""{}""#, json_escape(label))) + .unwrap_or_else(|| "null".to_string()); + let bbox = block + .bbox + .map(|[x0, y0, x1, y1]| { + format!(r#"{{"x0":{x0:.4},"y0":{y0:.4},"x1":{x1:.4},"y1":{y1:.4}}}"#) + }) + .unwrap_or_else(|| "null".to_string()); + format!( + r#"{{"type":"{}","label":{},"page":{},"bbox":{},"markdownSpan":[{},{}],"source":"{}","confidence":{{"layout":{},"ocr":{}}}}}"#, + block.block_type.as_str(), + label, + block.page, + bbox, + block.markdown_span.0, + block.markdown_span.1, + json_escape(&block.source), + optional_confidence_json(block.layout_confidence), + optional_confidence_json(block.ocr_confidence), + ) + }) + .collect::>() + .join(","); + format!( + r#"{{"schema_version":1,"pdf_type":"{}","page_count":{},"markdown":"{}","blocks":[{}]}}"#, + pdf_type, + result.page_count, + json_escape(&result.markdown), + blocks, + ) +} + #[cfg(test)] mod tests { use super::{extract_items_json, format_items_json, format_ocr_error_json}; @@ -403,6 +456,7 @@ fn main() { eprintln!("Options:"); eprintln!(" --json Output result as JSON"); eprintln!(" --items-json Output positioned TextItem JSON"); + eprintln!(" --layout-blocks-json Output Markdown plus typed layout blocks JSON"); eprintln!(" --raw Output only markdown (no headers)"); eprintln!( " --compact Collapse token-heavy source formatting such as dot leaders" @@ -424,6 +478,7 @@ fn main() { let pdf_path = &args[1]; let json_output = args.iter().any(|a| a == "--json"); let items_json_output = args.iter().any(|a| a == "--items-json"); + let layout_blocks_json_output = args.iter().any(|a| a == "--layout-blocks-json"); let raw_output = args.iter().any(|a| a == "--raw"); let compact_output = args.iter().any(|a| a == "--compact"); let page_numbers = args.iter().any(|a| a == "--pages"); @@ -485,9 +540,10 @@ fn main() { } if let Some(mode) = ocr_mode_argument { - if items_json_output || detect_only || analyze { + if items_json_output || layout_blocks_json_output || detect_only || analyze { exit_ocr_error( - "--ocr cannot be combined with --items-json, --detect-only, or --analyze", + "--ocr cannot be combined with --items-json, --layout-blocks-json, \ + --detect-only, or --analyze", json_output, ); } @@ -606,6 +662,27 @@ fn main() { return; } + if layout_blocks_json_output { + if page_filter.is_some() || password.is_some() || detect_only || analyze { + println!( + r#"{{"error":"{}"}}"#, + json_escape( + "--layout-blocks-json cannot be combined with --select-pages, \ + --password, --detect-only, or --analyze" + ) + ); + process::exit(1); + } + match pdf_inspector::extract_layout_blocks(pdf_path) { + Ok(result) => println!("{}", format_layout_blocks_json(&result)), + Err(e) => { + println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string())); + process::exit(1); + } + } + return; + } + let process_mode = if detect_only { ProcessMode::DetectOnly } else if analyze { diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index c6f046ae3..b314a3738 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -1272,7 +1272,7 @@ pub(crate) fn get_number(obj: &Object) -> Option { /// Visible page box: CropBox if present, else MediaBox, walking page-tree /// inheritance (both attributes are inheritable). Returns normalized /// (x0, y0, x1, y1) in PDF space. -fn get_page_box(doc: &Document, page_id: ObjectId) -> Option<(f32, f32, f32, f32)> { +pub(crate) fn get_page_box(doc: &Document, page_id: ObjectId) -> Option<(f32, f32, f32, f32)> { fn find_box(doc: &Document, page_id: ObjectId, key: &[u8]) -> Option> { let mut id = page_id; for _ in 0..32 { diff --git a/src/lib.rs b/src/lib.rs index d616cf6f3..404da3958 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -292,7 +292,7 @@ pub fn process_pdf_with_options>( let (doc, page_count) = load_document_from_path_with_password(&path, options.password.as_deref())?; - process_document(doc, page_count, options, start) + process_document(doc, page_count, options, start, None) } /// Process a PDF from a memory buffer with full extraction. @@ -318,7 +318,7 @@ pub fn process_pdf_mem_with_options( let (doc, page_count) = load_document_from_mem_with_password(buffer, options.password.as_deref())?; - process_document(doc, page_count, options, start) + process_document(doc, page_count, options, start, None) } // ========================================================================= @@ -963,6 +963,240 @@ pub fn extract_structure_elements>( extract_structure_elements_mem(&buffer, pages) } +// ========================================================================= +// Layout blocks (Markdown + typed blocks with spans, for citation grounding) +// ========================================================================= + +/// Hosted layout type of a [`LayoutBlock`]. +/// +/// Names follow the hosted layout-block vocabulary. Only the types the +/// native Full-mode pipeline actually emits are present: furniture (page +/// headers, footers, folios) is stripped before conversion and is never +/// invented here, and no ML layout model runs — everything is derived from +/// the same classification the Markdown convert loop already performs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LayoutBlockType { + /// Top-tier heading (`#`). + Title, + /// Lower-tier heading (`##`–`######`); [`LayoutBlock::label`] carries + /// the level ("H2".."H6"). + SectionHeader, + /// Body paragraph (also block quotes and short mono fragments demoted + /// to plain text). + Text, + /// Bulleted or numbered list item, including wrapped continuations. + ListItem, + /// Figure/table caption or source citation line. + Caption, + /// Fenced code block. + Code, + /// Markdown pipe table. + Table, + /// Image placeholder. The layout-blocks pipeline enables image + /// placeholders (unlike the default `process_pdf` Markdown, which + /// omits them), so figures surface as picture blocks with bboxes. + Picture, +} + +impl LayoutBlockType { + /// Hosted type name ("title", "section_header", "text", "list_item", + /// "caption", "code", "table", "picture"). + pub fn as_str(&self) -> &'static str { + match self { + LayoutBlockType::Title => "title", + LayoutBlockType::SectionHeader => "section_header", + LayoutBlockType::Text => "text", + LayoutBlockType::ListItem => "list_item", + LayoutBlockType::Caption => "caption", + LayoutBlockType::Code => "code", + LayoutBlockType::Table => "table", + LayoutBlockType::Picture => "picture", + } + } +} + +/// One typed layout block from the Full-mode Markdown pipeline. +/// +/// Produced by [`extract_layout_blocks`] / [`extract_layout_blocks_mem`]. +/// `markdown_span` grounds the block in the accompanying +/// [`LayoutBlocksResult::markdown`]; `bbox` grounds it on the source page. +#[derive(Debug, Clone)] +pub struct LayoutBlock { + /// Hosted block type. + pub block_type: LayoutBlockType, + /// Heading level label ("H2".."H6") for + /// [`LayoutBlockType::SectionHeader`], `None` otherwise. + pub label: Option, + /// 1-indexed page number the block was emitted from. + pub page: u32, + /// Normalized `[x0, y0, x1, y1]` in 0–1 page space with a top-left + /// origin (y grows downward), derived from the source `TextItem`, + /// table, or image coordinates. `None` when the page box is unknown or + /// the block has no positioned source items. + pub bbox: Option<[f32; 4]>, + /// `[start, end)` byte offsets into [`LayoutBlocksResult::markdown`]. + pub markdown_span: (usize, usize), + /// Provenance of the block's text. Always `"native_text"` for this + /// pipeline (no OCR, no layout model). + pub source: String, + /// Layout-model confidence. Always `None`: no layout model runs. + pub layout_confidence: Option, + /// OCR confidence. Always `None`: the text is native, not OCR output. + pub ocr_confidence: Option, +} + +/// Result of [`extract_layout_blocks`] / [`extract_layout_blocks_mem`]. +#[derive(Debug)] +pub struct LayoutBlocksResult { + /// Markdown assembled from the recorded fragments. Block spans index + /// into this string. Content matches the default `process_pdf` output + /// up to post-processing scope (cleanup passes run per fragment here so + /// spans stay exact; the default pipeline runs them document-wide). + pub markdown: String, + /// Typed blocks in reading order, with non-overlapping ascending spans. + pub blocks: Vec, + /// The detected PDF type. + pub pdf_type: PdfType, + /// Total page count. + pub page_count: u32, +} + +fn layout_block_type_and_label( + kind: markdown::blocks::RawBlockKind, +) -> (LayoutBlockType, Option) { + use markdown::blocks::RawBlockKind; + match kind { + RawBlockKind::Heading(1) => (LayoutBlockType::Title, None), + RawBlockKind::Heading(level) => ( + LayoutBlockType::SectionHeader, + Some(format!("H{}", level.clamp(2, 6))), + ), + RawBlockKind::Text => (LayoutBlockType::Text, None), + RawBlockKind::ListItem => (LayoutBlockType::ListItem, None), + RawBlockKind::Caption => (LayoutBlockType::Caption, None), + RawBlockKind::Code => (LayoutBlockType::Code, None), + RawBlockKind::Table => (LayoutBlockType::Table, None), + RawBlockKind::Picture => (LayoutBlockType::Picture, None), + } +} + +/// Normalize a PDF-space bbox (y-up) into 0–1 page space with a top-left +/// origin, clamped to the page box. +fn normalize_layout_bbox( + bbox: (f32, f32, f32, f32), + page_box: (f32, f32, f32, f32), +) -> Option<[f32; 4]> { + let (px0, py0, px1, py1) = page_box; + let width = px1 - px0; + let height = py1 - py0; + if !(width > 0.0 && height > 0.0) { + return None; + } + let x0 = ((bbox.0 - px0) / width).clamp(0.0, 1.0); + let x1 = ((bbox.2 - px0) / width).clamp(0.0, 1.0); + // Flip: PDF y grows upward, normalized page space grows downward. + let y0 = ((py1 - bbox.3) / height).clamp(0.0, 1.0); + let y1 = ((py1 - bbox.1) / height).clamp(0.0, 1.0); + Some([x0, y0, x1, y1]) +} + +/// Extract Firecrawl-compatible layout blocks from a PDF in memory. +/// +/// Runs the existing Full-mode extract+convert pipeline (same reading +/// order, heading classifier, table insertion, and furniture stripping as +/// [`process_pdf_mem`] — no layout model) and records each fragment the +/// Markdown convert loop emits as a typed [`LayoutBlock`] with: +/// +/// - `markdown_span`: exact `[start, end)` byte offsets into the returned +/// [`LayoutBlocksResult::markdown`], for citation grounding; +/// - `bbox`: normalized 0–1 page-space coordinates (top-left origin) from +/// the existing `TextItem` / table / image geometry; +/// - `source`: always `"native_text"`; confidence fields stay `None` +/// because no layout or OCR model produces a real score. +/// +/// Unlike the default `process_pdf` Markdown, this payload includes image +/// placeholders (`![Image: …](image)`) so figures surface as `picture` +/// blocks with page geometry. +/// +/// The default [`process_pdf_mem`] Markdown output is unaffected: recording +/// is a parallel observation of the same conversion pass. For Scanned or +/// ImageBased PDFs (or when extraction yields no trustworthy text) the +/// result carries the classification with empty `markdown` and `blocks`. +pub fn extract_layout_blocks_mem(buffer: &[u8]) -> Result { + let start = ProcessingTimer::start(); + validate_pdf_bytes(buffer)?; + let (doc, page_count) = load_document_from_mem(buffer)?; + + // Page boxes are needed for bbox normalization and the document is + // consumed by processing, so collect them first. + let page_boxes: HashMap = doc + .get_pages() + .iter() + .filter_map(|(&page, &page_id)| { + extractor::get_page_box(&doc, page_id).map(|page_box| (page, page_box)) + }) + .collect(); + + // Picture blocks require image placeholders in the conversion stream. + // `include_images` stays off in `MarkdownOptions::default()` so the + // default `process_pdf` output is unaffected; this opt-in payload is the + // layout view, where figures are part of the contract. + let mut options = PdfOptions::default(); + options.markdown.include_images = true; + + let mut sink: Option = None; + let result = process_document(doc, page_count, options, start, Some(&mut sink))?; + + // Scanned/ImageBased PDFs and garbage-text upgrades drop the Markdown; + // the blocks payload must follow the same trust decision. + if result.markdown.is_none() { + return Ok(LayoutBlocksResult { + markdown: String::new(), + blocks: Vec::new(), + pdf_type: result.pdf_type, + page_count: result.page_count, + }); + } + + let output = sink.unwrap_or_default(); + let blocks = output + .blocks + .into_iter() + .map(|block| { + let (block_type, label) = layout_block_type_and_label(block.kind); + let bbox = block.bbox.and_then(|bbox| { + page_boxes + .get(&block.page) + .and_then(|&page_box| normalize_layout_bbox(bbox, page_box)) + }); + LayoutBlock { + block_type, + label, + page: block.page, + bbox, + markdown_span: block.span, + source: "native_text".to_string(), + layout_confidence: None, + ocr_confidence: None, + } + }) + .collect(); + + Ok(LayoutBlocksResult { + markdown: output.markdown, + blocks, + pdf_type: result.pdf_type, + page_count: result.page_count, + }) +} + +/// Path-based wrapper for [`extract_layout_blocks_mem`]. +pub fn extract_layout_blocks>(path: P) -> Result { + validate_pdf_file(&path)?; + let buffer = std::fs::read(path.as_ref())?; + extract_layout_blocks_mem(&buffer) +} + // ========================================================================= // Region-based text extraction (for hybrid OCR pipelines) // ========================================================================= @@ -4163,11 +4397,17 @@ fn strip_leading_pdf_container_bytes(buf: &[u8]) -> Option> { } /// Core processing pipeline operating on a pre-loaded document. +/// +/// `layout_blocks` is an optional sink for layout-block recording (see +/// [`extract_layout_blocks_mem`]). When present and the Full-mode Markdown +/// conversion runs, it is filled with the recorded blocks payload; the +/// returned `markdown` stays byte-identical to a run without the sink. fn process_document( doc: Document, page_count: u32, options: PdfOptions, start: ProcessingTimer, + layout_blocks: Option<&mut Option>, ) -> Result { // Step 1 — Detection (cheap: scans content streams for text operators) let detection = detector::detect_from_document(&doc, page_count, &options.detection)?; @@ -4401,21 +4641,35 @@ fn process_document( let md = if options.mode == ProcessMode::Analyze { None } else { - Some(markdown::to_markdown_from_items_with_rects_and_lines( - items, - options.markdown, - &rects, - &lines, - markdown::MarkdownDocumentContext { - page_thresholds: &page_thresholds, - struct_roles: struct_roles.as_ref(), - struct_tables: &struct_tables, - page_count, - prefiltered_page_number_pages: Some(&removed_pages), - prefiltered_page_number_mask: Some(removal_mask.as_slice()), - precomputed_chart_regions: Some(&chart_regions), - }, - )) + let context = markdown::MarkdownDocumentContext { + page_thresholds: &page_thresholds, + struct_roles: struct_roles.as_ref(), + struct_tables: &struct_tables, + page_count, + prefiltered_page_number_pages: Some(&removed_pages), + prefiltered_page_number_mask: Some(removal_mask.as_slice()), + precomputed_chart_regions: Some(&chart_regions), + }; + if let Some(sink) = layout_blocks { + let (md, blocks) = + markdown::to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + items, + options.markdown, + &rects, + &lines, + context, + ); + *sink = Some(blocks); + Some(md) + } else { + Some(markdown::to_markdown_from_items_with_rects_and_lines( + items, + options.markdown, + &rects, + &lines, + context, + )) + } }; let enc = !ocr_reasons_by_page.is_empty() diff --git a/src/markdown/blocks.rs b/src/markdown/blocks.rs new file mode 100644 index 000000000..939a35aae --- /dev/null +++ b/src/markdown/blocks.rs @@ -0,0 +1,621 @@ +//! Layout-block recording for citation grounding. +//! +//! The Markdown convert loop already classifies every fragment it emits +//! (heading tier, list item, caption, code, table, image) — it just throws +//! the classification away once the text is pushed into the output string. +//! [`BlockRecorder`] captures those decisions as they happen: each emitted +//! fragment is recorded with its byte range in the raw (pre-postprocess) +//! output plus the union bbox of the source geometry. +//! +//! [`BlockRecorder::finish`] then assembles the final blocks payload. The +//! document-level postprocess pass ([`clean_markdown`]) rewrites bytes +//! (hyphenation, space collapsing, URL formatting), which would invalidate +//! recorded offsets — so each fragment is cleaned *individually* and the +//! payload's Markdown is reassembled from the cleaned fragments with the +//! original inter-block separators. `markdown_span` offsets are therefore +//! exact byte ranges into the payload's own `markdown` string. The default +//! `process_pdf` output is untouched: recording is opt-in and never changes +//! what the convert loop emits. + +use crate::types::{TextItem, TextLine}; + +use super::postprocess::clean_markdown; +use super::MarkdownOptions; + +/// Classification of one recorded fragment, mirroring the convert loop's +/// own emission branches. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RawBlockKind { + /// Markdown heading with its level (1–6). + Heading(usize), + /// Body paragraph, block quote, or short mono fragment demoted to text. + Text, + /// Bulleted or numbered list item (including wrapped continuations). + ListItem, + /// Figure/table caption or source citation line. + Caption, + /// Fenced code block. + Code, + /// Markdown pipe table. + Table, + /// Image placeholder (only emitted with `include_images`). + Picture, +} + +/// One block in the finalized payload: kind, source page, PDF-space bbox, +/// and its byte span in [`LayoutBlocksOutput::markdown`]. +#[derive(Debug, Clone)] +pub(crate) struct RecordedBlock { + pub(crate) kind: RawBlockKind, + /// 1-indexed page the block was emitted from. + pub(crate) page: u32, + /// Union bbox `(x0, y0, x1, y1)` in PDF points (y-up, bottom-left + /// origin), from the block's source items. + pub(crate) bbox: Option<(f32, f32, f32, f32)>, + /// `[start, end)` byte offsets into the finalized markdown. + pub(crate) span: (usize, usize), +} + +/// Finalized layout-blocks payload: the assembled markdown plus the blocks +/// whose `span` ranges index into it. +#[derive(Debug, Default)] +pub(crate) struct LayoutBlocksOutput { + pub(crate) markdown: String, + pub(crate) blocks: Vec, +} + +/// A fragment recorded against the raw convert-loop output. +#[derive(Debug, Clone)] +struct RawBlock { + kind: RawBlockKind, + page: u32, + start: usize, + end: usize, + bbox: Option<(f32, f32, f32, f32)>, +} + +/// Records emitted fragments during the convert loop. +#[derive(Debug, Default)] +pub(crate) struct BlockRecorder { + blocks: Vec, + /// Whether the last block may still be extended (open paragraph or + /// list item awaiting wrapped continuation lines). + open: bool, + finished: Option, +} + +impl BlockRecorder { + pub(crate) fn new() -> Self { + Self::default() + } + + /// Record one fragment emitted at `raw[start..end]`. + /// + /// With `continuation`, the fragment extends the currently open block of + /// the same kind (paragraph line joins, wrapped list items) instead of + /// starting a new one. `extendable` keeps the block open for future + /// continuations; one-shot emissions (headings, captions, tables) close + /// it immediately. + #[allow(clippy::too_many_arguments)] + pub(crate) fn push_fragment( + &mut self, + kind: RawBlockKind, + page: u32, + start: usize, + end: usize, + bbox: Option<(f32, f32, f32, f32)>, + continuation: bool, + extendable: bool, + ) { + if continuation && self.open { + if let Some(last) = self.blocks.last_mut() { + if last.kind == kind { + last.end = last.end.max(end); + // Blocks are page-scoped in practice; if a continuation + // crosses pages, keep the first page's geometry. + if last.page == page { + last.bbox = union_bbox(last.bbox, bbox); + } + self.open = extendable; + return; + } + } + } + self.blocks.push(RawBlock { + kind, + page, + start, + end, + bbox, + }); + self.open = extendable; + } + + /// Assemble the final payload from the raw convert-loop output. + /// + /// Each fragment is cleaned individually (same [`clean_markdown`] + /// passes as the default pipeline) so recorded offsets stay exact, and + /// fragments are re-joined with separators derived from the raw output + /// (single newline between adjacent list items, blank line between + /// paragraphs). Fragments that clean to nothing (e.g. a stray folio + /// removed by `remove_page_numbers`) are dropped. + pub(crate) fn finish(&mut self, raw: &str, options: &MarkdownOptions) { + let raw_blocks = std::mem::take(&mut self.blocks); + let mut markdown = String::new(); + let mut blocks = Vec::with_capacity(raw_blocks.len()); + // End of the previous block's trimmed content in the raw output; + // the bytes from here to the next block's trimmed start form the + // inter-block separator. + let mut prev_content_end = 0usize; + let mut have_prev = false; + + // `remove_page_numbers` decides by line isolation, and a single-item + // fragment is always isolated. Preserve page-number-shaped list items + // only when an adjacent list fragment on the same page makes them + // non-isolated in the document-level pass. + let list_item_options = MarkdownOptions { + remove_page_numbers: false, + ..options.clone() + }; + for (index, block) in raw_blocks.iter().enumerate() { + if block.start >= block.end || block.end > raw.len() { + continue; + } + let Some(fragment) = raw.get(block.start..block.end) else { + continue; + }; + let content_start = block.start + (fragment.len() - fragment.trim_start().len()); + let content_end = block.start + fragment.trim_end().len(); + let fragment_options = if block.kind == RawBlockKind::ListItem + && has_neighboring_list_content(&raw_blocks, index, raw) + { + &list_item_options + } else { + options + }; + let cleaned = clean_markdown(fragment.to_string(), fragment_options); + let cleaned = cleaned.trim(); + if cleaned.is_empty() { + // Removed entirely by postprocess (e.g. a folio line). + prev_content_end = content_end.max(prev_content_end); + continue; + } + + if have_prev { + let sep_src = raw + .get(prev_content_end.min(content_start)..content_start) + .unwrap_or(""); + if sep_src.chars().any(|c| !c.is_whitespace()) { + // Unrecorded non-whitespace between blocks (e.g. a + // `` marker) is preserved verbatim. + markdown.push_str("\n\n"); + markdown.push_str(sep_src.trim()); + markdown.push_str("\n\n"); + } else if sep_src.matches('\n').count() >= 2 { + markdown.push_str("\n\n"); + } else { + markdown.push('\n'); + } + } + + let span_start = markdown.len(); + markdown.push_str(cleaned); + blocks.push(RecordedBlock { + kind: block.kind, + page: block.page, + bbox: block.bbox, + span: (span_start, markdown.len()), + }); + prev_content_end = content_end; + have_prev = true; + } + + if !markdown.is_empty() { + markdown.push('\n'); + } + self.finished = Some(LayoutBlocksOutput { markdown, blocks }); + } + + /// Take the finalized payload (empty when the convert loop produced no + /// output at all). + pub(crate) fn take_output(&mut self) -> LayoutBlocksOutput { + self.finished.take().unwrap_or_default() + } +} + +fn has_neighboring_list_content(blocks: &[RawBlock], index: usize, raw: &str) -> bool { + let block = &blocks[index]; + let neighbors = [index.checked_sub(1), index.checked_add(1)]; + + neighbors.into_iter().flatten().any(|neighbor_index| { + let Some(neighbor) = blocks.get(neighbor_index) else { + return false; + }; + if neighbor.kind != RawBlockKind::ListItem || neighbor.page != block.page { + return false; + } + + let (left, right) = if neighbor_index < index { + (neighbor, block) + } else { + (block, neighbor) + }; + let Some(left_fragment) = raw.get(left.start..left.end) else { + return false; + }; + let Some(right_fragment) = raw.get(right.start..right.end) else { + return false; + }; + let left_content_end = left.start + left_fragment.trim_end().len(); + let right_content_start = + right.start + (right_fragment.len() - right_fragment.trim_start().len()); + raw.get(left_content_end..right_content_start) + .is_some_and(|separator| { + separator.chars().all(char::is_whitespace) && separator.matches('\n').count() == 1 + }) + }) +} + +fn union_bbox( + a: Option<(f32, f32, f32, f32)>, + b: Option<(f32, f32, f32, f32)>, +) -> Option<(f32, f32, f32, f32)> { + match (a, b) { + (Some(a), Some(b)) => Some((a.0.min(b.0), a.1.min(b.1), a.2.max(b.2), a.3.max(b.3))), + (Some(a), None) => Some(a), + (None, b) => b, + } +} + +fn item_bbox(item: &TextItem) -> Option<(f32, f32, f32, f32)> { + let x0 = item.x.min(item.x + item.width); + let x1 = item.x.max(item.x + item.width); + let y0 = item.y.min(item.y + item.height); + let y1 = item.y.max(item.y + item.height); + (x0.is_finite() && y0.is_finite() && x1.is_finite() && y1.is_finite()) + .then_some((x0, y0, x1, y1)) +} + +/// Union bbox of a text line's items in PDF points (y-up). +pub(crate) fn line_bbox(line: &TextLine) -> Option<(f32, f32, f32, f32)> { + line.items + .iter() + .fold(None, |acc, item| union_bbox(acc, item_bbox(item))) +} + +/// Fold a text line's bbox into an accumulator (used while buffering code +/// lines whose fenced block is emitted later). +pub(crate) fn union_line_bbox( + acc: Option<(f32, f32, f32, f32)>, + line: &TextLine, +) -> Option<(f32, f32, f32, f32)> { + union_bbox(acc, line_bbox(line)) +} + +/// Union bbox of the indexed items in PDF points (y-up). Used for table +/// blocks, whose `item_indices` reference the detection-time item slice. +pub(crate) fn items_bbox(items: &[TextItem], indices: &[usize]) -> Option<(f32, f32, f32, f32)> { + indices + .iter() + .filter_map(|&idx| items.get(idx)) + .fold(None, |acc, item| union_bbox(acc, item_bbox(item))) +} + +/// Bbox of a single (image) item in PDF points (y-up). +pub(crate) fn single_item_bbox(item: &TextItem) -> Option<(f32, f32, f32, f32)> { + item_bbox(item) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::markdown::{ + to_markdown_from_items_with_rects_and_lines, + to_markdown_with_layout_blocks_from_items_with_rects_and_lines, MarkdownDocumentContext, + MarkdownOptions, + }; + use crate::types::ItemType; + use std::collections::{HashMap, HashSet}; + + fn make_item(text: &str, x: f32, y: f32, font_size: f32, page: u32) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width: text.len() as f32 * font_size * 0.5, + height: font_size, + font: "Helvetica".to_string(), + font_tag: String::new(), + font_size, + page, + is_bold: false, + is_italic: false, + is_underline: false, + is_strikeout: false, + item_type: ItemType::Text, + mcid: None, + } + } + + fn context() -> MarkdownDocumentContext<'static> { + use once_cell::sync::Lazy; + static THRESHOLDS: Lazy> = Lazy::new(HashMap::new); + MarkdownDocumentContext { + page_thresholds: &THRESHOLDS, + struct_roles: None, + struct_tables: &[], + page_count: 1, + prefiltered_page_number_pages: None, + prefiltered_page_number_mask: None, + precomputed_chart_regions: None, + } + } + + fn sample_items() -> Vec { + vec![ + make_item("Big Title", 72.0, 720.0, 24.0, 1), + make_item("First paragraph line at body size.", 72.0, 680.0, 12.0, 1), + make_item( + "Second wrapped line of the paragraph.", + 72.0, + 666.0, + 12.0, + 1, + ), + make_item("• Item one", 72.0, 620.0, 12.0, 1), + make_item("• Item two", 72.0, 606.0, 12.0, 1), + make_item("Figure 1: A caption line.", 72.0, 560.0, 12.0, 1), + ] + } + + fn assert_spans_are_valid(markdown: &str, blocks: &[RecordedBlock]) { + let mut prev_end = 0usize; + for block in blocks { + let (start, end) = block.span; + assert!( + start >= prev_end && start < end && end <= markdown.len(), + "invalid span {:?} (prev_end {prev_end}) in {markdown:?}", + block.span + ); + assert!( + !markdown[start..end].trim().is_empty(), + "span {:?} slices to whitespace in {markdown:?}", + block.span + ); + prev_end = end; + } + } + + #[test] + fn records_headings_paragraphs_lists_and_captions() { + let (_, output) = to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + sample_items(), + MarkdownOptions::default(), + &[], + &[], + context(), + ); + let kinds: Vec = output.blocks.iter().map(|b| b.kind).collect(); + assert_eq!( + kinds, + vec![ + RawBlockKind::Heading(1), + RawBlockKind::Text, + RawBlockKind::ListItem, + RawBlockKind::ListItem, + RawBlockKind::Caption, + ], + "unexpected block kinds in {:?}", + output.markdown + ); + assert_spans_are_valid(&output.markdown, &output.blocks); + + let slice = |idx: usize| { + let (start, end) = output.blocks[idx].span; + &output.markdown[start..end] + }; + assert_eq!(slice(0), "# Big Title"); + assert_eq!( + slice(1), + "First paragraph line at body size. Second wrapped line of the paragraph." + ); + assert_eq!(slice(2), "- Item one"); + assert_eq!(slice(3), "- Item two"); + assert_eq!(slice(4), "Figure 1: A caption line."); + } + + #[test] + fn paragraph_bbox_unions_wrapped_lines() { + let (_, output) = to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + sample_items(), + MarkdownOptions::default(), + &[], + &[], + context(), + ); + let paragraph = &output.blocks[1]; + assert_eq!(paragraph.page, 1); + let (x0, y0, _, y1) = paragraph.bbox.expect("paragraph bbox"); + // Second line's baseline (666) through first line's top (680 + 12). + assert!(x0 <= 72.0 && y0 <= 666.0 && y1 >= 692.0, "{:?}", paragraph); + } + + #[test] + fn records_picture_blocks_when_images_are_included() { + let mut image = make_item("[Image: Im0]", 100.0, 400.0, 0.0, 1); + image.width = 200.0; + image.height = 150.0; + image.item_type = ItemType::Image; + let mut items = sample_items(); + items.push(image); + + let options = MarkdownOptions { + include_images: true, + ..MarkdownOptions::default() + }; + let (_, output) = to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + items, + options, + &[], + &[], + context(), + ); + let picture = output + .blocks + .iter() + .find(|b| b.kind == RawBlockKind::Picture) + .expect("picture block"); + assert_eq!(picture.bbox, Some((100.0, 400.0, 300.0, 550.0))); + let (start, end) = picture.span; + assert!(output.markdown[start..end].starts_with("![Image:")); + assert_spans_are_valid(&output.markdown, &output.blocks); + } + + #[test] + fn recording_does_not_change_default_markdown() { + let plain = to_markdown_from_items_with_rects_and_lines( + sample_items(), + MarkdownOptions::default(), + &[], + &[], + context(), + ); + let (recorded, _) = to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + sample_items(), + MarkdownOptions::default(), + &[], + &[], + context(), + ); + assert_eq!(plain, recorded); + } + + #[test] + fn finish_drops_fragments_that_clean_to_nothing() { + let mut recorder = BlockRecorder::new(); + let raw = "Hello world.\n\n17\n\nGoodbye.\n"; + recorder.push_fragment(RawBlockKind::Text, 1, 0, 12, None, false, true); + // A stray folio: `remove_page_numbers` cleans it to nothing. + recorder.push_fragment(RawBlockKind::Text, 1, 14, 16, None, false, true); + recorder.push_fragment(RawBlockKind::Text, 2, 18, 26, None, false, true); + recorder.finish(raw, &MarkdownOptions::default()); + let output = recorder.take_output(); + + assert_eq!(output.markdown, "Hello world.\n\nGoodbye.\n"); + assert_eq!(output.blocks.len(), 2); + assert_eq!(output.blocks[0].span, (0, 12)); + assert_eq!(output.blocks[1].span, (14, 22)); + assert_eq!(&output.markdown[14..22], "Goodbye."); + } + + #[test] + fn finish_keeps_page_number_shaped_list_items() { + // "- 5 -" is a page-number expression, but as a classified list item + // it is content. The document-level pass keeps it because its list + // neighbors break line isolation; the per-fragment pass must not + // drop it just because a lone fragment is always "isolated". + let mut recorder = BlockRecorder::new(); + let raw = "- one\n- 5 -\n- two\n"; + recorder.push_fragment(RawBlockKind::ListItem, 1, 0, 6, None, false, true); + recorder.push_fragment(RawBlockKind::ListItem, 1, 6, 12, None, false, true); + recorder.push_fragment(RawBlockKind::ListItem, 1, 12, 18, None, false, true); + recorder.finish(raw, &MarkdownOptions::default()); + let output = recorder.take_output(); + + assert_eq!(output.markdown, "- one\n- 5 -\n- two\n"); + assert_eq!(output.blocks.len(), 3); + assert_eq!(&output.markdown[6..11], "- 5 -"); + } + + #[test] + fn finish_drops_isolated_page_number_shaped_list_item() { + let mut recorder = BlockRecorder::new(); + let raw = "- one\n\n- 5 -\n\nAfter.\n"; + recorder.push_fragment(RawBlockKind::ListItem, 1, 0, 6, None, false, true); + recorder.push_fragment(RawBlockKind::ListItem, 1, 7, 13, None, false, true); + recorder.push_fragment(RawBlockKind::Text, 1, 14, 20, None, false, true); + recorder.finish(raw, &MarkdownOptions::default()); + let output = recorder.take_output(); + + assert_eq!(output.markdown, "- one\n\nAfter.\n"); + assert_eq!(output.blocks.len(), 2); + } + + #[test] + fn finish_preserves_single_newline_list_separators() { + let mut recorder = BlockRecorder::new(); + let raw = "- one\n- two\n\nAfter list.\n"; + recorder.push_fragment(RawBlockKind::ListItem, 1, 0, 6, None, false, true); + recorder.push_fragment(RawBlockKind::ListItem, 1, 6, 12, None, false, true); + recorder.push_fragment(RawBlockKind::Text, 1, 13, 24, None, false, true); + recorder.finish(raw, &MarkdownOptions::default()); + let output = recorder.take_output(); + + assert_eq!(output.markdown, "- one\n- two\n\nAfter list.\n"); + assert_spans_are_valid(&output.markdown, &output.blocks); + assert_eq!(&output.markdown[6..11], "- two"); + } + + #[test] + fn continuation_extends_the_open_block() { + let mut recorder = BlockRecorder::new(); + let raw = "- item wraps here\n"; + recorder.push_fragment( + RawBlockKind::ListItem, + 1, + 0, + 7, + Some((72.0, 600.0, 120.0, 612.0)), + false, + true, + ); + recorder.push_fragment( + RawBlockKind::ListItem, + 1, + 7, + 18, + Some((90.0, 586.0, 150.0, 598.0)), + true, + true, + ); + recorder.finish(raw, &MarkdownOptions::default()); + let output = recorder.take_output(); + + assert_eq!(output.blocks.len(), 1); + assert_eq!(output.blocks[0].bbox, Some((72.0, 586.0, 150.0, 612.0))); + assert_eq!( + &output.markdown[output.blocks[0].span.0..output.blocks[0].span.1], + "- item wraps here" + ); + } + + #[test] + fn unrecorded_page_markers_are_preserved_as_separators() { + let mut recorder = BlockRecorder::new(); + let raw = "First page text.\n\n\n\n\n\nSecond page text.\n"; + recorder.push_fragment(RawBlockKind::Text, 1, 0, 16, None, false, true); + recorder.push_fragment(RawBlockKind::Text, 2, 37, 54, None, false, true); + recorder.finish(raw, &MarkdownOptions::default()); + let output = recorder.take_output(); + + assert_eq!( + output.markdown, + "First page text.\n\n\n\nSecond page text.\n" + ); + assert_spans_are_valid(&output.markdown, &output.blocks); + } + + #[test] + fn empty_input_yields_empty_output() { + let (_, output) = to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + Vec::new(), + MarkdownOptions::default(), + &[], + &[], + context(), + ); + assert!(output.markdown.is_empty()); + assert!(output.blocks.is_empty()); + let _ = HashSet::::new(); + } +} diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 4b016de9b..dc09ff058 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -11,6 +11,7 @@ use super::analysis::{ detect_header_level, font_size_rarity, has_dot_leaders, is_heading_fragment, is_toc_entry_line, is_toc_marker_heading, }; +use super::blocks::{line_bbox, BlockRecorder, RawBlockKind}; use super::classify::{format_list_item, is_caption_line, is_list_item, starts_with_bullet_marker}; use super::heading::classify_heading_sequences; use super::postprocess::clean_markdown; @@ -44,6 +45,9 @@ pub(super) struct PositionedMarkdown { x: f32, markdown: String, chart_order: Option, + /// Union bbox `(x0, y0, x1, y1)` of the block's source items in PDF + /// points (y-up). Only consumed by layout-block recording. + bbox: Option<(f32, f32, f32, f32)>, } impl PositionedMarkdown { @@ -58,8 +62,14 @@ impl PositionedMarkdown { x, markdown, chart_order, + bbox: None, } } + + pub(super) fn with_bbox(mut self, bbox: Option<(f32, f32, f32, f32)>) -> Self { + self.bbox = bbox; + self + } } fn chart_stream_position( @@ -637,7 +647,15 @@ fn count_table_columns(table_md: &str) -> usize { 0 } +fn positioned_block_kind(kind: PositionedBlockKind) -> RawBlockKind { + match kind { + PositionedBlockKind::Table => RawBlockKind::Table, + PositionedBlockKind::Image => RawBlockKind::Picture, + } +} + /// Flush any remaining tables and images for a given page +#[allow(clippy::too_many_arguments)] fn flush_page_tables_and_images( page: u32, page_blocks: &HashMap>>, @@ -645,6 +663,7 @@ fn flush_page_tables_and_images( inserted_images: &mut HashSet<(u32, usize)>, output: &mut String, in_paragraph: &mut bool, + recorder: &mut Option<&mut BlockRecorder>, ) { let Some(blocks) = page_blocks.get(&page) else { return; @@ -662,7 +681,19 @@ fn flush_page_tables_and_images( *in_paragraph = false; } output.push('\n'); + let frag_start = output.len(); output.push_str(&block.markdown); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + positioned_block_kind(kind), + page, + frag_start, + output.len(), + block.bbox, + false, + false, + ); + } output.push('\n'); match kind { PositionedBlockKind::Table => { @@ -676,6 +707,7 @@ fn flush_page_tables_and_images( } /// Convert text lines to markdown, inserting tables and images at appropriate Y positions +#[cfg(test)] pub(super) fn to_markdown_from_lines_with_tables_and_images( lines: Vec, options: MarkdownOptions, @@ -686,6 +718,38 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( struct_roles: Option< &std::collections::HashMap>, >, +) -> String { + to_markdown_from_lines_with_tables_and_images_recorded( + lines, + options, + page_tables, + page_images, + page_chart_regions, + band_split_pages, + struct_roles, + None, + ) +} + +/// [`to_markdown_from_lines_with_tables_and_images`] with optional layout +/// block recording. With `recorder: None` the behavior — including the +/// returned Markdown bytes — is identical to the plain function. With a +/// recorder, every emitted fragment is additionally recorded and the +/// recorder is finished against the raw output before postprocessing, so +/// the caller can take a [`super::blocks::LayoutBlocksOutput`] whose spans +/// are exact. +#[allow(clippy::too_many_arguments)] +pub(super) fn to_markdown_from_lines_with_tables_and_images_recorded( + lines: Vec, + options: MarkdownOptions, + page_tables: std::collections::HashMap>, + page_images: std::collections::HashMap>, + page_chart_regions: &std::collections::HashMap>, + band_split_pages: &HashSet, + struct_roles: Option< + &std::collections::HashMap>, + >, + mut recorder: Option<&mut BlockRecorder>, ) -> String { if lines.is_empty() && page_tables.is_empty() && page_images.is_empty() { return String::new(); @@ -772,25 +836,50 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let mut last_list_x: Option = None; // Code lines accumulate here and the fence is emitted only when the // block flushes with content — an empty ``` ``` pair can never appear. - fn flush_code_block(output: &mut String, pending_code: &mut String) { + fn flush_code_block( + output: &mut String, + pending_code: &mut String, + recorder: &mut Option<&mut BlockRecorder>, + page: u32, + bbox: &mut Option<(f32, f32, f32, f32)>, + ) { let trimmed = pending_code.trim(); + let frag_start = output.len(); // A fragment too short to be code — a lone ® or stray glyph set in // a mono face — reads better as plain text than as a fenced block. - if trimmed.chars().count() < 3 { + let kind = if trimmed.chars().count() < 3 { if !trimmed.is_empty() { output.push_str(trimmed); output.push_str("\n\n"); } + RawBlockKind::Text } else { output.push_str("```\n"); output.push_str(pending_code); output.push_str("```\n"); + RawBlockKind::Code + }; + if output.len() > frag_start { + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + kind, + page, + frag_start, + output.len(), + bbox.take(), + false, + false, + ); + } } + *bbox = None; pending_code.clear(); } let mut in_code_block = false; let mut pending_code = String::new(); + let mut pending_code_page = 0u32; + let mut pending_code_bbox: Option<(f32, f32, f32, f32)> = None; let mut prev_had_dot_leaders = false; let mut paragraph_in_wrapped_bold_run = false; let mut toc_suppress_page: Option = None; @@ -824,7 +913,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Flush current page's remaining tables and images if current_page > 0 { if in_code_block { - flush_code_block(&mut output, &mut pending_code); + flush_code_block( + &mut output, + &mut pending_code, + &mut recorder, + pending_code_page, + &mut pending_code_bbox, + ); in_code_block = false; } flush_page_tables_and_images( @@ -834,6 +929,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( &mut inserted_images, &mut output, &mut in_paragraph, + &mut recorder, ); if in_paragraph { output.push_str("\n\n"); @@ -858,6 +954,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( &mut inserted_images, &mut output, &mut in_paragraph, + &mut recorder, ); if in_paragraph { output.push_str("\n\n"); @@ -891,7 +988,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // precedes it in reading order. A code line after the // block reopens a new fence naturally. if in_code_block { - flush_code_block(&mut output, &mut pending_code); + flush_code_block( + &mut output, + &mut pending_code, + &mut recorder, + pending_code_page, + &mut pending_code_bbox, + ); in_code_block = false; } if in_paragraph { @@ -900,7 +1003,19 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( paragraph_in_wrapped_bold_run = false; } output.push('\n'); + let frag_start = output.len(); output.push_str(&block.markdown); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + positioned_block_kind(kind), + current_page, + frag_start, + output.len(), + block.bbox, + false, + false, + ); + } output.push('\n'); match kind { PositionedBlockKind::Table => { @@ -979,7 +1094,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Close code block when transitioning to non-code if in_code_block && !is_code_line { - flush_code_block(&mut output, &mut pending_code); + flush_code_block( + &mut output, + &mut pending_code, + &mut recorder, + pending_code_page, + &mut pending_code_bbox, + ); in_code_block = false; } @@ -993,7 +1114,19 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( in_paragraph = false; paragraph_in_wrapped_bold_run = false; } + let frag_start = output.len(); output.push_str(trimmed); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::Caption, + line.page, + frag_start, + output.len(), + line_bbox(line), + false, + false, + ); + } output.push_str("\n\n"); continue; } @@ -1117,7 +1250,19 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( } else { plain_text.clone() }; + let frag_start = output.len(); output.push_str(&format!("{} {}\n\n", prefix, heading_text.trim())); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::Heading(level), + line.page, + frag_start, + output.len(), + line_bbox(line), + false, + false, + ); + } if is_toc_marker_heading(plain_trimmed) { toc_suppress_page = Some(line.page); } @@ -1141,8 +1286,20 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( in_paragraph = false; paragraph_in_wrapped_bold_run = false; } + let frag_start = output.len(); output.push_str(&format!("- {}", trimmed)); output.push('\n'); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::ListItem, + line.page, + frag_start, + output.len(), + line_bbox(line), + false, + true, + ); + } in_list = true; last_list_x = line.items.first().map(|i| i.x); continue; @@ -1156,8 +1313,20 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( paragraph_in_wrapped_bold_run = false; } let formatted = format_list_item(trimmed); + let frag_start = output.len(); output.push_str(&formatted); output.push('\n'); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::ListItem, + line.page, + frag_start, + output.len(), + line_bbox(line), + false, + true, + ); + } in_list = true; last_list_x = line.items.first().map(|i| i.x); continue; @@ -1183,8 +1352,20 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( output.pop(); output.push(' '); } + let frag_start = output.len(); output.push_str(trimmed); output.push('\n'); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::ListItem, + line.page, + frag_start, + output.len(), + line_bbox(line), + true, + true, + ); + } continue; } else { in_list = false; @@ -1202,7 +1383,19 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( in_paragraph = false; paragraph_in_wrapped_bold_run = false; } + let frag_start = output.len(); output.push_str(&format!("> {}\n", trimmed)); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::Text, + line.page, + frag_start, + output.len(), + line_bbox(line), + false, + false, + ); + } continue; } @@ -1213,6 +1406,10 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( in_paragraph = false; paragraph_in_wrapped_bold_run = false; } + if !in_code_block { + pending_code_page = line.page; + } + pending_code_bbox = super::blocks::union_line_bbox(pending_code_bbox, line); in_code_block = true; pending_code.push_str(plain_trimmed); pending_code.push('\n'); @@ -1221,6 +1418,8 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Regular text - join lines within same paragraph with space let cur_dot_leaders = has_dot_leaders(plain_trimmed); + let was_in_paragraph = in_paragraph; + let frag_start = output.len(); if in_paragraph { if cur_dot_leaders || prev_had_dot_leaders { output.push('\n'); @@ -1229,6 +1428,17 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( } } output.push_str(trimmed); + if let Some(r) = recorder.as_deref_mut() { + r.push_fragment( + RawBlockKind::Text, + line.page, + frag_start, + output.len(), + line_bbox(line), + was_in_paragraph, + true, + ); + } paragraph_in_wrapped_bold_run = if in_paragraph { paragraph_in_wrapped_bold_run || line_in_wrapped_bold_run } else { @@ -1240,7 +1450,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Close any trailing code block if in_code_block { - flush_code_block(&mut output, &mut pending_code); + flush_code_block( + &mut output, + &mut pending_code, + &mut recorder, + pending_code_page, + &mut pending_code_bbox, + ); } // Flush current page and any remaining pages with tables/images @@ -1252,6 +1468,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( &mut inserted_images, &mut output, &mut in_paragraph, + &mut recorder, ); for &p in &all_content_pages { if p <= current_page { @@ -1264,6 +1481,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( &mut inserted_images, &mut output, &mut in_paragraph, + &mut recorder, ); } @@ -1272,6 +1490,13 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( output.push('\n'); } + // Blocks are finished against the raw output; per-fragment cleaning in + // `BlockRecorder::finish` keeps their spans exact while the default + // return value below goes through the usual document-level postprocess. + if let Some(r) = recorder { + r.finish(&output, &options); + } + // Clean up and post-process clean_markdown(output, &options) } diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs index 2c5c9607e..1f2345a71 100644 --- a/src/markdown/mod.rs +++ b/src/markdown/mod.rs @@ -7,6 +7,7 @@ //! - Paragraphs pub(crate) mod analysis; +pub(crate) mod blocks; mod classify; mod convert; mod furniture; @@ -21,10 +22,11 @@ use std::collections::{HashMap, HashSet}; use crate::types::{PdfLine, PdfRect, TextItem}; use analysis::calculate_font_stats_from_items; +use blocks::BlockRecorder; use classify::{format_list_item, is_caption_line, is_code_like, is_list_item}; use convert::{ - merge_continuation_tables, to_markdown_from_lines_with_tables_and_images, ChartProseOrder, - PositionedMarkdown, + merge_continuation_tables, to_markdown_from_lines_with_tables_and_images_recorded, + ChartProseOrder, PositionedMarkdown, }; const CHART_REGION_PAD: f32 = 20.0; @@ -874,20 +876,21 @@ impl TableDetectionOutput { page: u32, table: &crate::tables::Table, chart_order: Option, + items: &[TextItem], ) { self.pages_with_detected_tables.insert(page); match self.mode { TableOutputMode::Markdown => { self.pages_with_tables.insert(page); - self.markdown_by_page - .entry(page) - .or_default() - .push(PositionedMarkdown::new( + self.markdown_by_page.entry(page).or_default().push( + PositionedMarkdown::new( table.rows.first().copied().unwrap_or(0.0), table.columns.first().copied().unwrap_or(0.0), crate::tables::table_to_markdown(table), chart_order, - )); + ) + .with_bbox(blocks::items_bbox(items, &table.item_indices)), + ); } #[cfg(feature = "ocr")] TableOutputMode::CompleteTables => { @@ -1468,10 +1471,38 @@ pub(crate) fn to_markdown_from_items_with_rects_and_lines( pdf_lines, context, TableOutputMode::Markdown, + None, ) .markdown } +/// Run the ordinary Markdown pipeline while recording layout blocks. +/// +/// Returns the standard document-level Markdown (byte-identical to +/// [`to_markdown_from_items_with_rects_and_lines`]) plus a +/// [`blocks::LayoutBlocksOutput`] whose `markdown` is reassembled from the +/// same emitted fragments with exact `[start, end)` spans per block. +pub(crate) fn to_markdown_with_layout_blocks_from_items_with_rects_and_lines( + items: Vec, + options: MarkdownOptions, + rects: &[crate::types::PdfRect], + pdf_lines: &[crate::types::PdfLine], + context: MarkdownDocumentContext<'_>, +) -> (String, blocks::LayoutBlocksOutput) { + let mut recorder = BlockRecorder::new(); + let markdown = convert_items_with_rects_lines_and_table_output( + items, + options, + rects, + pdf_lines, + context, + TableOutputMode::Markdown, + Some(&mut recorder), + ) + .markdown; + (markdown, recorder.take_output()) +} + /// Run the ordinary Markdown table pipeline but return only structurally /// complete data tables. Supplemental OCR uses this to keep detector behavior /// identical without parsing the serialized Markdown back into tables. @@ -1496,6 +1527,7 @@ pub(crate) fn complete_table_markdown_from_items( precomputed_chart_regions: None, }, TableOutputMode::CompleteTables, + None, ); let mut output = String::new(); for (_, table) in conversion.detected_tables { @@ -1518,6 +1550,7 @@ fn convert_items_with_rects_lines_and_table_output( pdf_lines: &[crate::types::PdfLine], context: MarkdownDocumentContext<'_>, table_output_mode: TableOutputMode, + recorder: Option<&mut BlockRecorder>, ) -> MarkdownConversionOutput { use crate::tables::{ content_width, detect_tables_from_lines, detect_tables_from_rects, @@ -1807,7 +1840,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, table, chart_prose_order); + table_output.record(page, table, chart_prose_order, band_items); } } @@ -1831,7 +1864,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, table, chart_prose_order); + table_output.record(page, table, chart_prose_order, band_items); } // 2. Line-based detection on unclaimed items (when rects didn't find tables) @@ -1846,7 +1879,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, table, chart_prose_order); + table_output.record(page, table, chart_prose_order, band_items); } } @@ -1882,7 +1915,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, &table, chart_prose_order); + table_output.record(page, &table, chart_prose_order, &inside_items); for &band_idx in &inside_map { rect_claimed.insert(band_idx); } @@ -1940,7 +1973,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, &table, chart_prose_order); + table_output.record(page, &table, chart_prose_order, subset_items); } }; @@ -2002,7 +2035,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, &table, chart_prose_order); + table_output.record(page, &table, chart_prose_order, band_items); } } } @@ -2061,7 +2094,7 @@ fn convert_items_with_rects_lines_and_table_output( table_items.insert(global_idx); } } - table_output.record(page, table, chart_prose_order); + table_output.record(page, table, chart_prose_order, &page_text); } } } @@ -2127,7 +2160,7 @@ fn convert_items_with_rects_lines_and_table_output( } } } - table_output.record(page, table, chart_prose_order); + table_output.record(page, table, chart_prose_order, &chart_free); } } } @@ -2148,15 +2181,15 @@ fn convert_items_with_rects_lines_and_table_output( .and_then(|s| s.strip_suffix(']')) .unwrap_or(&img.text); let img_md = format!("![Image: {}](image)\n", img_name); - page_images - .entry(img.page) - .or_default() - .push(PositionedMarkdown::new( + page_images.entry(img.page).or_default().push( + PositionedMarkdown::new( img.y, img.x, img_md, page_chart_prose_orders.get(&img.page).copied(), - )); + ) + .with_bbox(blocks::single_item_bbox(img)), + ); } // Check structure tree coverage on ALL text items (before table filtering) @@ -2395,7 +2428,7 @@ fn convert_items_with_rects_lines_and_table_output( // Convert to markdown, inserting tables and images at appropriate positions let mut band_split_page_set: HashSet = page_band_splits.keys().copied().collect(); band_split_page_set.extend(page_chart_prose_splits.keys().copied()); - let markdown = to_markdown_from_lines_with_tables_and_images( + let markdown = to_markdown_from_lines_with_tables_and_images_recorded( lines, options, page_tables, @@ -2403,6 +2436,7 @@ fn convert_items_with_rects_lines_and_table_output( &page_chart_map, &band_split_page_set, effective_struct_roles, + recorder, ); MarkdownConversionOutput { markdown, @@ -2427,7 +2461,7 @@ mod tests { vec![vec!["header a".into(), "header b".into()]], vec![0, 1], ); - output.record(1, &incomplete, None); + output.record(1, &incomplete, None, &[]); assert!(output.has_detected_tables_on_page(1)); assert!(!output.has_tables_on_page(1)); assert!(output.complete_tables.is_empty()); @@ -2441,7 +2475,7 @@ mod tests { ], vec![0, 1, 2, 3], ); - output.record(1, &complete, None); + output.record(1, &complete, None, &[]); assert!(output.has_tables_on_page(1)); assert_eq!(output.complete_tables.len(), 1); } diff --git a/src/python.rs b/src/python.rs index 44141bd09..5d5c8df0d 100644 --- a/src/python.rs +++ b/src/python.rs @@ -411,6 +411,79 @@ pub struct PyStructureElement { pub role: String, } +/// One typed layout block from the Full-mode Markdown pipeline. +#[pyclass(name = "LayoutBlock")] +#[derive(Clone)] +pub struct PyLayoutBlock { + /// Hosted block type: "title", "section_header", "text", "list_item", + /// "caption", "code", "table", or "picture". + #[pyo3(get)] + pub block_type: String, + /// Heading level label ("H2".."H6") for section_header blocks, None + /// otherwise. + #[pyo3(get)] + pub label: Option, + /// 1-indexed page number the block was emitted from. + #[pyo3(get)] + pub page: u32, + /// Normalized (x0, y0, x1, y1) in 0-1 page space with a top-left + /// origin, or None when no positioned geometry is available. + #[pyo3(get)] + pub bbox: Option<(f32, f32, f32, f32)>, + /// [start, end) byte offsets into the result's markdown. + #[pyo3(get)] + pub markdown_span: (usize, usize), + /// Provenance of the block's text; always "native_text". + #[pyo3(get)] + pub source: String, + /// Layout-model confidence; always None (no layout model runs). + #[pyo3(get)] + pub layout_confidence: Option, + /// OCR confidence; always None (the text is native, not OCR output). + #[pyo3(get)] + pub ocr_confidence: Option, +} + +#[pymethods] +impl PyLayoutBlock { + fn __repr__(&self) -> String { + format!( + "LayoutBlock(type='{}', page={}, span={:?})", + self.block_type, self.page, self.markdown_span + ) + } +} + +/// Markdown plus typed layout blocks whose spans index into it. +#[pyclass(name = "LayoutBlocksResult")] +pub struct PyLayoutBlocksResult { + /// Markdown assembled from the recorded fragments; block spans are + /// byte offsets into this string. + #[pyo3(get)] + pub markdown: String, + /// Typed blocks in reading order with non-overlapping ascending spans. + #[pyo3(get)] + pub blocks: Vec, + /// 'text_based', 'scanned', 'image_based', or 'mixed'. + #[pyo3(get)] + pub pdf_type: String, + /// Total page count. + #[pyo3(get)] + pub page_count: u32, +} + +#[pymethods] +impl PyLayoutBlocksResult { + fn __repr__(&self) -> String { + format!( + "LayoutBlocksResult(pdf_type='{}', pages={}, blocks={})", + self.pdf_type, + self.page_count, + self.blocks.len() + ) + } +} + #[pymethods] impl PyStructureElement { fn __repr__(&self) -> String { @@ -603,6 +676,28 @@ fn convert_structure_elements(elements: Vec) -> Vec PyLayoutBlocksResult { + PyLayoutBlocksResult { + markdown: result.markdown, + blocks: result + .blocks + .into_iter() + .map(|block| PyLayoutBlock { + block_type: block.block_type.as_str().to_string(), + label: block.label, + page: block.page, + bbox: block.bbox.map(|[x0, y0, x1, y1]| (x0, y0, x1, y1)), + markdown_span: block.markdown_span, + source: block.source, + layout_confidence: block.layout_confidence, + ocr_confidence: block.ocr_confidence, + }) + .collect(), + pdf_type: pdf_type_str(result.pdf_type), + page_count: result.page_count, + } +} + fn parse_page_regions( page_regions: Vec<(u32, Vec>)>, ) -> PyResult)>> { @@ -991,6 +1086,40 @@ fn extract_structure_elements_bytes( Ok(convert_structure_elements(elements)) } +/// Extract Markdown plus typed layout blocks from a PDF file. +/// +/// Runs the same Full-mode extract+convert pipeline as [`process_pdf`] +/// (no layout model) and records each emitted fragment as a typed block +/// with a normalized 0-1 page-space bbox (top-left origin) and an exact +/// [start, end) byte span into the returned markdown — enough to ground +/// citations both in the text and on the page. +/// +/// Unlike the default process_pdf markdown, the payload includes image +/// placeholders so figures surface as "picture" blocks. +/// +/// For scanned/image-based PDFs the result carries the classification with +/// empty markdown and no blocks. +/// +/// Args: +/// path: Path to the PDF file. +/// +/// Returns: +/// LayoutBlocksResult with markdown, blocks, pdf_type, and page_count. +#[pyfunction] +fn extract_layout_blocks(path: &str) -> PyResult { + let result = crate::extract_layout_blocks(path).map_err(to_py_err)?; + Ok(to_py_layout_blocks_result(result)) +} + +/// Extract Markdown plus typed layout blocks from PDF bytes. +/// +/// See [`extract_layout_blocks`] for details. +#[pyfunction] +fn extract_layout_blocks_bytes(data: &[u8]) -> PyResult { + let result = crate::extract_layout_blocks_mem(data).map_err(to_py_err)?; + Ok(to_py_layout_blocks_result(result)) +} + /// Python module definition. #[pymodule] fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -1004,6 +1133,8 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -1022,6 +1153,8 @@ fn pdf_inspector(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(extract_text_with_positions_bytes, m)?)?; m.add_function(wrap_pyfunction!(extract_structure_elements, m)?)?; m.add_function(wrap_pyfunction!(extract_structure_elements_bytes, m)?)?; + m.add_function(wrap_pyfunction!(extract_layout_blocks, m)?)?; + m.add_function(wrap_pyfunction!(extract_layout_blocks_bytes, m)?)?; m.add_function(wrap_pyfunction!(extract_text_in_regions, m)?)?; m.add_function(wrap_pyfunction!(extract_text_in_regions_bytes, m)?)?; m.add_function(wrap_pyfunction!(extract_pages_markdown, m)?)?; diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 9c30aecb5..7e3885650 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1532,6 +1532,295 @@ fn test_extract_structure_elements_untagged_pdf_empty() { ); } +// --------------------------------------------------------------------------- +// extract_layout_blocks / extract_layout_blocks_mem +// --------------------------------------------------------------------------- + +/// Spans must be ascending, non-overlapping, in-bounds byte ranges whose +/// slices carry non-whitespace content. +fn assert_layout_block_spans_valid(result: &pdf_inspector::LayoutBlocksResult) { + let markdown = result.markdown.as_bytes(); + let mut prev_end = 0usize; + for block in &result.blocks { + let (start, end) = block.markdown_span; + assert!( + start >= prev_end && start < end && end <= markdown.len(), + "invalid span {:?} after {} in {} bytes of markdown", + block.markdown_span, + prev_end, + markdown.len() + ); + let slice = std::str::from_utf8(&markdown[start..end]).expect("span on char boundary"); + assert!( + !slice.trim().is_empty(), + "span {:?} slices to whitespace", + block.markdown_span + ); + prev_end = end; + } +} + +/// Simple document built with lopdf: a large title, two body lines, and +/// two bullet list items. +fn synthetic_layout_blocks_pdf() -> Vec { + use lopdf::content::{Content, Operation}; + use lopdf::{dictionary, Document, Object, Stream}; + + let mut doc = Document::with_version("1.5"); + let pages_id = doc.new_object_id(); + let page_id = doc.new_object_id(); + let font_id = doc.new_object_id(); + let content_id = doc.new_object_id(); + + doc.objects.insert( + font_id, + dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + } + .into(), + ); + + let text = |ops: &mut Vec, x: i64, y: i64, size: i64, s: &str| { + ops.push(Operation::new("BT", vec![])); + ops.push(Operation::new("Tf", vec!["F1".into(), size.into()])); + ops.push(Operation::new("Td", vec![x.into(), y.into()])); + ops.push(Operation::new("Tj", vec![Object::string_literal(s)])); + ops.push(Operation::new("ET", vec![])); + }; + + let mut operations = Vec::new(); + text(&mut operations, 72, 720, 24, "Annual Report"); + text( + &mut operations, + 72, + 676, + 11, + "This is the opening paragraph of the report body text.", + ); + text( + &mut operations, + 72, + 662, + 11, + "It continues on a second wrapped line for the same paragraph.", + ); + text(&mut operations, 72, 620, 11, "- First finding of the year"); + text(&mut operations, 72, 606, 11, "- Second finding of the year"); + + let content = Content { operations }.encode().unwrap(); + doc.objects + .insert(content_id, Stream::new(dictionary! {}, content).into()); + + doc.objects.insert( + page_id, + dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + "Resources" => dictionary! { + "Font" => dictionary! { + "F1" => font_id, + }, + }, + "Contents" => content_id, + } + .into(), + ); + doc.objects.insert( + pages_id, + dictionary! { + "Type" => "Pages", + "Kids" => vec![page_id.into()], + "Count" => 1, + } + .into(), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).unwrap(); + bytes +} + +#[test] +fn test_extract_layout_blocks_synthetic_pdf() { + use pdf_inspector::LayoutBlockType; + + let buf = synthetic_layout_blocks_pdf(); + let result = pdf_inspector::extract_layout_blocks_mem(&buf).unwrap(); + + assert_eq!(result.page_count, 1); + assert!(!result.markdown.is_empty()); + assert_layout_block_spans_valid(&result); + + let title = result + .blocks + .iter() + .find(|b| b.block_type == LayoutBlockType::Title) + .expect("title block"); + let (start, end) = title.markdown_span; + assert_eq!(&result.markdown[start..end], "# Annual Report"); + assert_eq!(title.block_type.as_str(), "title"); + + let list_items: Vec<_> = result + .blocks + .iter() + .filter(|b| b.block_type == LayoutBlockType::ListItem) + .collect(); + assert_eq!(list_items.len(), 2, "blocks: {:?}", result.blocks); + let (start, end) = list_items[0].markdown_span; + assert_eq!(&result.markdown[start..end], "- First finding of the year"); + + let paragraph = result + .blocks + .iter() + .find(|b| b.block_type == LayoutBlockType::Text) + .expect("text block"); + let (start, end) = paragraph.markdown_span; + assert!( + result.markdown[start..end].contains("second wrapped line"), + "wrapped lines should merge into one text block: {:?}", + &result.markdown[start..end] + ); + + for block in &result.blocks { + assert_eq!(block.page, 1); + assert_eq!(block.source, "native_text"); + assert!(block.layout_confidence.is_none()); + assert!(block.ocr_confidence.is_none()); + let [x0, y0, x1, y1] = block.bbox.expect("synthetic text has geometry"); + assert!( + (0.0..=1.0).contains(&x0) + && (0.0..=1.0).contains(&y0) + && (0.0..=1.0).contains(&y1) + && x0 <= x1 + && y0 <= y1 + && x1 <= 1.0, + "bbox must be normalized 0-1: {:?}", + block.bbox + ); + } + + // Top-left origin: the 24pt title at the top of the page must have a + // smaller normalized y than the list items near the middle. + let list_bbox = list_items[0].bbox.unwrap(); + assert!( + title.bbox.unwrap()[1] < list_bbox[1], + "title should sit above the list in top-left page space" + ); +} + +#[test] +fn test_extract_layout_blocks_fixture_grounds_native_text() { + use pdf_inspector::LayoutBlockType; + + let result = pdf_inspector::extract_layout_blocks("tests/fixtures/thermo-freon12.pdf").unwrap(); + assert_eq!(result.pdf_type, pdf_inspector::PdfType::TextBased); + assert_eq!(result.page_count, 3); + assert!(!result.blocks.is_empty()); + assert_layout_block_spans_valid(&result); + + // The blocks must cover essentially all of the emitted markdown — + // only inter-block separators are outside spans. + let covered: usize = result + .blocks + .iter() + .map(|b| b.markdown_span.1 - b.markdown_span.0) + .sum(); + assert!( + covered * 10 >= result.markdown.len() * 9, + "blocks should cover >=90% of the markdown ({} of {})", + covered, + result.markdown.len() + ); + + // This fixture has ruled tables; they must surface as table blocks + // whose spans slice to pipe tables. + let table = result + .blocks + .iter() + .find(|b| b.block_type == LayoutBlockType::Table) + .expect("table block"); + let (start, end) = table.markdown_span; + assert!(result.markdown[start..end].contains('|')); + + // Section headers carry their heading level as a label. + let header = result + .blocks + .iter() + .find(|b| b.block_type == LayoutBlockType::SectionHeader) + .expect("section_header block"); + assert!(header + .label + .as_deref() + .is_some_and(|label| label.starts_with('H'))); + + for block in &result.blocks { + assert!((1..=3).contains(&block.page)); + if let Some([x0, y0, x1, y1]) = block.bbox { + assert!((0.0..=1.0).contains(&x0) && x0 <= x1 && x1 <= 1.0); + assert!((0.0..=1.0).contains(&y0) && y0 <= y1 && y1 <= 1.0); + } + } +} + +#[test] +fn test_extract_layout_blocks_markdown_matches_default_output() { + // Per-fragment postprocess can in principle diverge from the + // document-level pass at fragment boundaries, but on this snapshot + // fixture the two must agree byte-for-byte — a strong regression + // signal that recording never changes what the convert loop emits. + // The layout payload enables image placeholders, so compare against a + // run with the same option (the `process_pdf_mem` default stays + // image-free and is covered by the snapshot tests). + let buf = std::fs::read("tests/fixtures/thermo-freon12.pdf").unwrap(); + let mut options = pdf_inspector::PdfOptions::default(); + options.markdown.include_images = true; + let default = pdf_inspector::process_pdf_mem_with_options(&buf, options) + .unwrap() + .markdown + .unwrap(); + let blocks = pdf_inspector::extract_layout_blocks_mem(&buf).unwrap(); + assert_eq!(default, blocks.markdown); +} + +#[test] +fn test_extract_layout_blocks_surfaces_picture_blocks() { + use pdf_inspector::LayoutBlockType; + + // The layout payload enables image placeholders (unlike the default + // process_pdf markdown), so figures must surface as picture blocks. + let result = + pdf_inspector::extract_layout_blocks("tests/fixtures/text_page_with_watermark_image.pdf") + .unwrap(); + assert_layout_block_spans_valid(&result); + let picture = result + .blocks + .iter() + .find(|b| b.block_type == LayoutBlockType::Picture) + .expect("picture block"); + let (start, end) = picture.markdown_span; + assert!(result.markdown[start..end].starts_with("![Image:")); + let [x0, y0, x1, y1] = picture.bbox.expect("image geometry"); + assert!((0.0..=1.0).contains(&x0) && x0 <= x1 && x1 <= 1.0); + assert!((0.0..=1.0).contains(&y0) && y0 <= y1 && y1 <= 1.0); +} + +#[test] +fn test_extract_layout_blocks_image_based_pdf_is_empty() { + let buf = std::fs::read("tests/fixtures/scan_with_native_header_text.pdf").unwrap(); + let result = pdf_inspector::extract_layout_blocks_mem(&buf).unwrap(); + assert_eq!(result.pdf_type, pdf_inspector::PdfType::ImageBased); + assert!(result.markdown.is_empty()); + assert!(result.blocks.is_empty()); +} + #[test] fn test_identity_h_no_tounicode_suppresses_garbage() { // shinagawa_identity_h.pdf uses YuGothic with Identity-H encoding and no diff --git a/tests/test_python.py b/tests/test_python.py index 0ea7c5169..2d1879151 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -321,6 +321,110 @@ def test_not_a_pdf(self): pdf_inspector.extract_structure_elements_bytes(b"not a pdf") +# --------------------------------------------------------------------------- +# extract_layout_blocks / extract_layout_blocks_bytes +# --------------------------------------------------------------------------- + + +ALLOWED_BLOCK_TYPES = { + "title", + "section_header", + "text", + "list_item", + "caption", + "code", + "table", + "picture", +} + + +class TestExtractLayoutBlocks: + def test_file(self): + result = pdf_inspector.extract_layout_blocks( + fixture_path("thermo-freon12.pdf") + ) + assert result.pdf_type == "text_based" + assert result.page_count == 3 + assert len(result.markdown) > 0 + assert len(result.blocks) > 0 + assert all(b.block_type in ALLOWED_BLOCK_TYPES for b in result.blocks) + + def test_spans_ground_the_markdown(self): + result = pdf_inspector.extract_layout_blocks( + fixture_path("thermo-freon12.pdf") + ) + md = result.markdown.encode("utf-8") + prev_end = 0 + for block in result.blocks: + start, end = block.markdown_span + assert 0 <= start < end <= len(md) + assert start >= prev_end # non-overlapping, ascending + assert md[start:end].decode("utf-8").strip() + prev_end = end + + def test_bbox_normalized(self): + result = pdf_inspector.extract_layout_blocks( + fixture_path("thermo-freon12.pdf") + ) + assert any(b.bbox is not None for b in result.blocks) + for block in result.blocks: + assert 1 <= block.page <= result.page_count + if block.bbox is None: + continue + x0, y0, x1, y1 = block.bbox + assert 0.0 <= x0 <= x1 <= 1.0 + assert 0.0 <= y0 <= y1 <= 1.0 + + def test_provenance(self): + result = pdf_inspector.extract_layout_blocks( + fixture_path("thermo-freon12.pdf") + ) + assert all(b.source == "native_text" for b in result.blocks) + assert all(b.layout_confidence is None for b in result.blocks) + assert all(b.ocr_confidence is None for b in result.blocks) + headers = [b for b in result.blocks if b.block_type == "section_header"] + assert all( + b.label is not None and b.label.startswith("H") for b in headers + ) + + def test_table_blocks_detected(self): + result = pdf_inspector.extract_layout_blocks( + fixture_path("thermo-freon12.pdf") + ) + tables = [b for b in result.blocks if b.block_type == "table"] + assert len(tables) > 0 + start, end = tables[0].markdown_span + assert "|" in result.markdown.encode("utf-8")[start:end].decode("utf-8") + + def test_picture_blocks_included(self): + # The layout payload enables image placeholders, so figures surface + # as picture blocks (the default process_pdf markdown omits them). + result = pdf_inspector.extract_layout_blocks( + fixture_path("text_page_with_watermark_image.pdf") + ) + pictures = [b for b in result.blocks if b.block_type == "picture"] + assert len(pictures) > 0 + start, end = pictures[0].markdown_span + md = result.markdown.encode("utf-8") + assert md[start:end].decode("utf-8").startswith("![Image:") + + def test_bytes(self): + data = fixture_bytes("thermo-freon12.pdf") + result = pdf_inspector.extract_layout_blocks_bytes(data) + assert len(result.blocks) > 0 + + def test_repr(self): + result = pdf_inspector.extract_layout_blocks( + fixture_path("thermo-freon12.pdf") + ) + assert "LayoutBlocksResult" in repr(result) + assert "LayoutBlock" in repr(result.blocks[0]) + + def test_not_a_pdf(self): + with pytest.raises(ValueError): + pdf_inspector.extract_layout_blocks_bytes(b"not a pdf") + + # --------------------------------------------------------------------------- # extract_text_in_regions / extract_text_in_regions_bytes # ---------------------------------------------------------------------------