import time
import logging
from pathlib import Path
from typing import List, Dict, Any, Optional
from PIL import Image
import threading

import pypdfium2 as pdfium
from google import genai
from google.genai import types

from .ocr_markdown import normalize_note_tables
from ..config import DEFAULT_MODEL, PAGES_DIR, BACKEND_DIR, gemini_api_key
from ..rate_limiter import execute_with_retry

logger = logging.getLogger("ocr.ocr_service")

def load_ocr_system_prompt() -> str:
    """Loads system prompt from OCR Instruction.md, or falls back to standard instructions."""
    instruction_path = BACKEND_DIR / "OCR Instruction.md"
    if instruction_path.exists():
        try:
            with open(instruction_path, "r", encoding="utf-8") as f:
                content = f.read()
                # Remove any trailing warnings or meta-instructions
                if "DO NOT WRITE ANY SCRIPTS" in content:
                    content = content.split("DO NOT WRITE ANY SCRIPTS")[0].strip()
                return content
        except Exception as e:
            logger.warning(f"Error reading OCR Instruction.md: {e}")

    return """Extract the contents of this image as clean Markdown, exactly as seen on the document.
- Transcribe all text accurately, preserving original structure, headers, and lists.
- ALWAYS capture and retain all important date, period, currency, and unit-wise information printed on the page:
  * Reporting dates, statement periods, approval/signing dates, and effective dates (e.g. "As at 31 December 2024", "For the year ended 31 December 2024", "Date: 15 March 2025"). NEVER omit dates even if printed near signature blocks or margins.
  * Reporting unit, currency, scale multiplier, denomination, or per-share basis (e.g. "(Amount in BDT)", "(Figures in thousands)", "(In Millions)", "Taka") directly above or preceding the table so scale, unit, and temporal context are never lost.
- EXCLUDE strictly non-content layout elements:
  * Page numbers (e.g. isolated page numbers printed in header or footer margins).
  * Signatures, handwritten notes, and signature stamps.
  * Names and designations of signatories (e.g. Chairman, CEO, Managing Director, CFO, Company Secretary, or Director signature blocks). Note: retain any approval or signing dates associated with them.
  * Graphs, charts, diagrams, and illustrations (do not describe or transcribe them).
- Output only the raw extracted content, with no preamble, introduction, or commentary.
- Extract tables as valid Markdown tables with a header separator row (e.g. |---|---|).
- Convert table-like structures, forms, and grids into flat tables.

- STRICT PHYSICAL COLUMN ORDERING (LEFT-TO-RIGHT) — NEVER SWAP COLUMNS:
  1. PRIMARY FINANCIAL STATEMENTS (Balance Sheet / Statement of Financial Position, Income Statement / Profit & Loss, Statement of Cash Flows, Statement of Changes in Equity):
     * The physical visual layout from left to right is ALWAYS:
       `| Particulars | Note | <Year 1> | <Year 2> | ... |` (e.g. `| Particulars | Note | 2024 | 2023 |`).
     * CRITICAL: Particulars / line item descriptions are VISUALLY on the LEFT. Therefore, `Particulars` MUST ALWAYS be Column 1.
     * The `Note` column is in the MIDDLE (between Particulars and the Year columns). Therefore, `Note` MUST ALWAYS be Column 2.
     * Financial amounts are on the RIGHT. Therefore, `<Year 1>` and `<Year 2>` MUST be Columns 3, 4, etc.
     * ABSOLUTE PROHIBITION: NEVER, UNDER ANY CIRCUMSTANCES, PLACE THE `Note` COLUMN AS COLUMN 1 ON A PRIMARY STATEMENT! Writing `| Note | Particulars | ... |` when Particulars is on the left is a severe hallucination that causes column shift and catastrophic data loss.
  2. SCHEDULES & NOTES PAGES:
     * ONLY if a detailed disclosure table physically prints reference numbers or note codes to the left of the item text (e.g. Column 1 physically printed as "Note No." followed by Column 2 "Description") may `| Note | Particulars | ... |` be used.
     * Whenever item descriptions appear on the left, `Particulars` MUST BE Column 1!
  3. Hierarchical / Grouped Column Headers (Consolidated vs. Standalone / Operating Segments):
     * When a table features multi-level column headers spanning entities, scopes, or segments (e.g. Consolidated vs. Standalone), bake the entity or segment descriptor directly into each corresponding child column header:
       `| Particulars | Note | Consolidated 2024 | Consolidated 2023 | Standalone 2024 | Standalone 2023 |`
     * NEVER produce duplicate, ambiguous column headers like `| 2024 | 2023 | 2024 | 2023 |`.

- STRICT LINE ITEM LABELS & SECTION IDENTIFIERS (A, B, C, 1, 2, i, ii):
  1. Section headers, category letters, and row identifiers such as `A)`, `B)`, `C)`, `D)`, `E)`, `F)`, `G)`, `(i)`, `(ii)`, `1.`, `2.` ARE PART OF THE LINE ITEM DESCRIPTION.
  2. They MUST be transcribed inside the `Particulars` column together with the line item title:
     - `| **A) Cash flows from operating activities** | | | |`
     - `| **(i) Operating expenses** | | 1,250,000 | 1,100,000 |`
     - `| Cash receipts from customers | | 50,000,000 | 42,000,000 |`
     - `| Payments to suppliers and employees | 14.1 | (28,500,000) | (24,100,000) |`
  3. ABSOLUTE PROHIBITION: NEVER PUT SECTION LETTERS ('A)', 'B)', 'C)') OR LINE NUMBER INDEXES INTO THE `Note` COLUMN!
     The `Note` column is RESERVED EXCLUSIVELY for actual accounting disclosure note numbers (e.g. `4`, `5`, `12`, `14.1`, `23`).

- STRICT TABLE INTEGRITY AND ZERO-DRIFT VALUE ALIGNMENT (MANDATORY):
  1. EVERY SINGLE ROW in a table MUST contain the EXACT SAME number of cells (pipes) as the table header row.
     If the header has 4 columns: `| Particulars | Note | 2024 | 2023 |`, every row MUST contain exactly 4 cells:
     `| <Particulars text> | <Note or blank> | <Year 1 Value or blank> | <Year 2 Value or blank> |`
  2. When a row has a note reference number (e.g. Note 5 for Property, plant and equipment, Note 12 for Trade payables, Note 18 for Retained earnings):
     - The note number belongs strictly in Column 2 (`Note`).
     - Example: `| **Property, plant and equipment** | 5 | 12,500,000 | 10,200,000 |`
     - Example: `| **Trade and other payables** | 12 | 8,400,000 | 7,100,000 |`
     - Example: `| **Long-term borrowings** | 15 | 25,000,000 | 20,000,000 |`
  3. When a row does NOT have a note reference number:
     - Leave Column 2 (`Note`) completely blank: `| |`.
     - Example: `| Finished goods inventory | | 3,450,000 | 2,800,000 |`
     - Example: `| Prepaid expenses and advances | | 420,000 | 380,000 |`
  4. ZERO-DRIFT PROHIBITIONS:
     - NEVER output 5 cells on a 4-column table!
     - NEVER put the Note reference number into a financial value column!
     - NEVER push Year 1 amounts into the Year 2 column!
     - All financial values for Year 1 must strictly align under the Year 1 header, and all financial values for Year 2 must strictly align under the Year 2 header!

- STRICT INFERRED PARTICULARS FOR TOTAL / SUB-TOTAL ROWS (NEVER LEAVE PARTICULARS BLANK):
  1. ABSOLUTE PROHIBITION: NEVER leave the `Particulars` column blank on any row containing financial figures!
     Lines like `| | | 12,500,000 | 10,200,000 |` are strictly forbidden.
  2. Financial statements often print group sub-totals or category sums as numbers at the bottom of an indented block without repeating the line item text.
  3. For ANY row that contains numeric values but has a blank, missing, or omitted line item description on the physical page:
     - You MUST populate the `Particulars` column with `**Total**` or `**Total <Category>**` (e.g. `| **Total** | | 12,500,000 | 10,200,000 |` or `| **Total Current Assets** | | 12,500,000 | 10,200,000 |`).
     - Simply putting `**Total**` or `**Subtotal**` ensures every numeric row has an identifiable line item label, preventing orphan data cells and extraction loss.

- ZERO-MERGE NUMERIC RULE — ONE FINANCIAL AMOUNT PER CELL (MANDATORY):
  1. ABSOLUTE PROHIBITION: NEVER combine multiple financial figures or group subtotals into a single cell separated by a space!
     - Cells containing two numbers (e.g. `| 15,200,000 23,500,000 |` or `| 4,100,000 9,800,000 |`) are STRICTLY FORBIDDEN.
     - Every single financial figure MUST reside in its own dedicated cell separated by `|`.
  2. Handling Underlined Group Sub-totals and Sums:
     - When a statement prints line items followed by an underline and an unlabeled subtotal amount below them:
       DO NOT glue the subtotal amount into the line item row above it!
       Instead, emit the subtotal as a SEPARATE NEW ROW directly underneath, populated with an inferred label such as `**Total <Category>**` or `**Subtotal**`.
     - Standard Example:
       ```markdown
       | Trade receivables | 6.1 | 14,500,000 | 11,200,000 |
       | Other current receivables | 6.2 | 5,300,000 | 4,100,000 |
       | **Total Receivables** | | 19,800,000 | 15,300,000 |
       ```
     - STRICTLY FORBIDDEN (DO NOT GLUE SUB-TOTAL TO LINE ITEM):
       ```markdown
       | Other current receivables | 6.2 | 5,300,000 19,800,000 | 4,100,000 15,300,000 |
       ```
"""

OCR_SYSTEM_PROMPT = load_ocr_system_prompt()

def get_genai_client() -> genai.Client:
    """Gemini client of the OCR step (OCR_API_KEY)."""
    key = gemini_api_key("ocr")
    if not key:
        raise ValueError("OCR_API_KEY is not configured.")
    return genai.Client(api_key=key)

# PDFium is not thread-safe: pages of one request are OCR'd in parallel, so every PDF open / render goes
# through this lock (a concurrent open fails with "Failed to load page" / "Data format error").
PDF_LOCK = threading.RLock()


def render_pdf_to_images(
    pdf_path: str,
    instrument_code: str,
    fiscal_year: str,
    dpi: int = 150
) -> List[Dict[str, Any]]:
    """
    Renders PDF pages to PNG files on disk using pypdfium2.
    Returns list of page information dictionaries.
    """
    code = instrument_code.upper().strip()
    out_dir = PAGES_DIR / code / fiscal_year
    out_dir.mkdir(parents=True, exist_ok=True)

    pages_info = []
    scale = dpi / 72.0
    with PDF_LOCK:
        pdf = pdfium.PdfDocument(pdf_path)
        try:
            total_pages = len(pdf)
            logger.info(f"Rendering {total_pages} pages from '{pdf_path}' to {out_dir}...")
            for idx, page in enumerate(pdf):
                page_num = idx + 1
                img_filename = f"page_{page_num}.png"
                img_dest = out_dir / img_filename
                pil_image = page.render(scale=scale).to_pil()
                pil_image.save(img_dest, format="PNG", optimize=True)
                pages_info.append({
                    "page_number": page_num,
                    "filename": img_filename,
                    "image_path": str(img_dest),
                    "width": pil_image.width,
                    "height": pil_image.height
                })
        finally:
            pdf.close()

    logger.info(f"Successfully rendered {len(pages_info)} page images for {code} / {fiscal_year}")
    return pages_info

def render_single_pdf_page(
    pdf_path: str,
    instrument_code: str,
    fiscal_year: str,
    page_number: int,
    dpi: int = 150
) -> Dict[str, Any]:
    """Renders only a single specific page from a PDF on demand (fast, ~30-50ms)."""
    code = instrument_code.upper().strip()
    out_dir = PAGES_DIR / code / fiscal_year
    out_dir.mkdir(parents=True, exist_ok=True)
    img_dest = out_dir / f"page_{page_number}.png"

    with PDF_LOCK:
        pdf = pdfium.PdfDocument(pdf_path)
        try:
            total_pages = len(pdf)
            if page_number < 1 or page_number > total_pages:
                raise ValueError(f"Page number {page_number} out of bounds (1-{total_pages})")
            pil_image = pdf[page_number - 1].render(scale=dpi / 72.0).to_pil()
        finally:
            pdf.close()
    pil_image.save(img_dest, format="PNG")

    return {
        "page_number": page_number,
        "image_path": str(img_dest),
        "width": pil_image.width,
        "height": pil_image.height
    }

def page_source_line(context: Optional[Dict[str, Any]]) -> str:
    """The first line of every OCR page: which company, security code, report year and PDF page it is."""
    if not context:
        return ""
    name, code = context.get("company_name") or "", context.get("security_code") or ""
    who = f"{name} ({code})" if name and code else (name or code)
    parts = [p for p in (who, f"FY {context['fiscal_year']}" if context.get("fiscal_year") else "",
                         f"Page {context['page_number']}" if context.get("page_number") else "") if p]
    return "Source: " + " | ".join(parts) if parts else ""


def run_gemini_ocr_on_image(
    image_path: str,
    model_name: str = DEFAULT_MODEL,
    context: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
    """
    Sends an individual document page image to Gemini 3.5 Flash-Lite and extracts Markdown.
    Guaranteed to respect the Free Tier rate limit.
    context: {company_name, security_code, fiscal_year, page_number} of the page; the model is told
    which report page it reads and writes the `Source:` line (page_source_line) as the first line.
    """
    client = get_genai_client()
    system_prompt = load_ocr_system_prompt()

    if not Path(image_path).exists():
        raise FileNotFoundError(f"Image not found: {image_path}")

    pil_img = Image.open(image_path)

    source_line = page_source_line(context)

    def _call_api():
        start_time = time.time()
        user_prompt = (
            "Transcribe the tabular financial data and text from this annual report page into clean Markdown format according to the instructions.\n"
            "MANDATORY COLUMN RULES:\n"
            "1. Physical visual order is strictly preserved from left to right. On primary financial statements (Balance Sheet, Income Statement, Cash Flow, Changes in Equity), 'Particulars' is the leftmost column (Column 1), 'Note' is in the middle (Column 2), and year amounts are on the right (| Particulars | Note | <Year 1> | <Year 2> |). NEVER place 'Note' to the left of 'Particulars'.\n"
            "2. Section identifiers like 'A)', 'B)', 'C)', '(i)', '(ii)', '1.', '2.' belong in 'Particulars' with the line item description. NEVER put 'A)', 'B)', 'C)' in the 'Note' column.\n"
            "3. Note numbers (e.g. 5, 12, 15.1) belong strictly in Column 2 ('Note'). Never put note numbers in a year column, and never shift columns.\n"
            "4. NEVER leave 'Particulars' blank on a row with numeric figures! For subtotal or total rows where the description is omitted on the page, infer and write '**Total**' or '**Total <Category>**' (e.g. '| **Total** | | 12,500,000 | 10,200,000 |') so every numeric row has an identifiable label.\n"
            "5. ZERO-MERGE NUMERIC RULE: NEVER combine multiple numbers or group subtotals into a single cell separated by a space (e.g. '| 15,200,000 23,500,000 |' is strictly forbidden). Every financial figure must have its own dedicated cell separated by '|'. Unlabeled subtotals under an underline must be emitted as a separate new row directly below, labeled '**Total <Category>**' or '**Subtotal**'.\n"
            "6. NOTES PAGES: one table per note. A note or sub-note number with its title (e.g. '12.1 Trade receivables') is a Markdown heading line ('### 12.1 Trade receivables') above its own table, never a table row; repeat the year header row in every note's table."
        )
        if source_line:
            user_prompt = (
                f"PAGE CONTEXT (given by the system): company {context.get('company_name') or '-'}, security code "
                f"{context.get('security_code') or '-'}, annual report for fiscal year {context.get('fiscal_year') or '-'}, "
                f"PDF page {context.get('page_number') or '-'}.\n"
                f"Write exactly this line as the FIRST line of your Markdown, then a blank line, then the page:\n{source_line}\n\n"
                + user_prompt
            )
        response = client.models.generate_content(
            model=model_name,
            contents=[pil_img, user_prompt],
            config=types.GenerateContentConfig(
                system_instruction=system_prompt,
                temperature=0.1,
            )
        )
        duration_sec = time.time() - start_time

        prompt_tokens = 0
        candidates_tokens = 0
        total_tokens = 0
        if hasattr(response, "usage_metadata") and response.usage_metadata:
            usage = response.usage_metadata
            prompt_tokens = getattr(usage, "prompt_token_count", 0) or 0
            candidates_tokens = getattr(usage, "candidates_token_count", 0) or 0
            total_tokens = getattr(usage, "total_token_count", 0) or (prompt_tokens + candidates_tokens)

        raw_text = response.text or ""
        if not raw_text and hasattr(response, "candidates") and response.candidates:
            for cand in response.candidates:
                if hasattr(cand, "content") and cand.content:
                    parts = getattr(cand.content, "parts", None)
                    if parts:
                        for part in parts:
                            if hasattr(part, "text") and part.text:
                                raw_text += part.text

        # If recitation filter was triggered or empty, retry with a structured JSON table extraction to bypass copyright recitation filter
        # and preserve statement title, hierarchy, and multiplier context at top of table
        if not raw_text.strip():
            logger.info("Recitation or empty text detected. Retrying with structured JSON table extraction including multiplier & hierarchy metadata...")
            try:
                json_prompt = (
                    "Extract all financial statements, tables, and numeric data from this document image into a structured JSON object.\n"
                    "Preserve all data required for downstream extraction according to these rules:\n\n"
                    "1. STATEMENT METADATA:\n"
                    "   - statement_title: string or null (e.g. 'CONSOLIDATED STATEMENT OF FINANCIAL POSITION', 'NOTES TO THE ACCOUNTS')\n"
                    "   - reporting_period: string or null (e.g. 'As at 31 December 2024', 'For the year ended 31 December 2024')\n"
                    "   - currency: string or null (e.g. 'USD', 'EUR', 'GBP', or local currency code)\n"
                    "   - multiplier_context: string or null (scale description e.g. 'Figures in Thousands', 'In Millions')\n"
                    "   - multiplier_value: number (1.0 for units, 1000 for thousand, 1000000 for million)\n\n"
                    "2. TABLES STRUCTURE & COLUMN INTEGRITY:\n"
                    "   - tables: array of table objects (or top-level 'columns' and 'rows'):\n"
                    "     * title: string or null (table heading or subheader, if applicable)\n"
                    "     * columns: array of string column names, preserving physical layout printed on the page:\n"
                    "       - When Particulars appears before Notes: ['Particulars', 'Notes', ...]\n"
                    "       - When Notes appears before Particulars (disclosure/notes pages): ['Notes', 'Particulars', ...]\n"
                    "       - Include 'Notes' in columns if note reference numbers (e.g. 4, 12, 18.1) appear anywhere in rows.\n"
                    "       - For hierarchical/multi-entity columns (Consolidated vs. Standalone / Operating Segments), bake the entity or segment descriptor directly into the column names (e.g. ['Consolidated 2024', 'Consolidated 2023', 'Standalone 2024', 'Standalone 2023']). Never output ambiguous duplicate year headers.\n"
                    "     * rows: array of row objects where keys match the exact column names (or array of cell string lists matching the columns):\n"
                    "       - Section labels and item prefixes (e.g. 'A)', '1.', '(i)', '(ii)') belong in 'Particulars' with the line item description.\n"
                    "       - Note reference numbers belong strictly in 'Notes' (leave empty '' if a row lacks a note).\n"
                    "       - If a row has numeric values but a blank description, infer a bracketed label like '[Total Non Current Assets]'.\n"
                    "       - Every row MUST provide cell values for all columns.\n\n"
                    "3. EXCLUSIONS:\n"
                    "   - Exclude isolated page numbers, handwritten signatures, corporate signatory name blocks, and graphs/charts."
                )
                retry_resp = client.models.generate_content(
                    model=model_name,
                    contents=[pil_img, json_prompt],
                    config=types.GenerateContentConfig(
                        response_mime_type="application/json",
                        temperature=0.1,
                    )
                )
                if retry_resp.text:
                    import json
                    parsed_json = json.loads(retry_resp.text)

                    top_lines = []
                    raw_tables_data = []

                    if isinstance(parsed_json, dict):
                        # Extract title
                        title = parsed_json.get("statement_title") or parsed_json.get("document_title")
                        if title:
                            top_lines.append(f"# {title}")

                        # Extract metadata badges
                        meta_badges = []
                        period = parsed_json.get("reporting_period") or parsed_json.get("period")
                        if period:
                            meta_badges.append(f"**Reporting Period:** {period}")

                        curr = parsed_json.get("currency")
                        mult_ctx = parsed_json.get("multiplier_context")
                        mult_val = parsed_json.get("multiplier_value")
                        if curr or mult_ctx or mult_val is not None:
                            curr_display = f"{curr} ({mult_ctx})" if (curr and mult_ctx) else (curr or mult_ctx or "Standard Units")
                            val_display = mult_val if mult_val is not None else 1.0
                            meta_badges.append(f"**Reporting Currency & Multiplier:** {curr_display} | **Scale Multiplier:** {val_display}")

                        if meta_badges:
                            top_lines.append("  \n".join(meta_badges))

                        # Extract tables list
                        if "tables" in parsed_json and isinstance(parsed_json["tables"], list):
                            raw_tables_data = parsed_json["tables"]
                        elif "rows" in parsed_json or "columns" in parsed_json:
                            raw_tables_data = [{
                                "title": parsed_json.get("title"),
                                "columns": parsed_json.get("columns"),
                                "rows": parsed_json.get("rows", [])
                            }]
                        else:
                            # Search for any list value that looks like rows
                            for k, v in parsed_json.items():
                                if isinstance(v, list) and v and isinstance(v[0], (dict, list)):
                                    raw_tables_data = [{"columns": None, "rows": v}]
                                    break
                    elif isinstance(parsed_json, list):
                        raw_tables_data = [{"columns": None, "rows": parsed_json}]

                    md_tables = []
                    for t_idx, tbl in enumerate(raw_tables_data):
                        if not isinstance(tbl, dict):
                            continue
                        tbl_title = tbl.get("title")
                        tbl_cols = tbl.get("columns")
                        tbl_rows = tbl.get("rows") or []

                        if not isinstance(tbl_rows, list) or len(tbl_rows) == 0:
                            continue

                        # Determine columns/headers
                        headers = []
                        if isinstance(tbl_cols, list) and len(tbl_cols) > 0:
                            headers = [str(c).strip() for c in tbl_cols if str(c).strip()]

                        # If columns not explicitly provided, infer from all rows
                        if not headers:
                            ordered_keys = []
                            seen_keys = set()
                            for r in tbl_rows:
                                if isinstance(r, dict):
                                    for k in r.keys():
                                        if k not in seen_keys:
                                            seen_keys.add(k)
                                            ordered_keys.append(k)
                            headers = [k.replace("_", " ").title() if "_" in k else k for k in ordered_keys]
                            key_map = dict(zip(headers, ordered_keys))
                        else:
                            key_map = None

                        if not headers:
                            continue

                        # Build table header & separator rows
                        header_row = "| " + " | ".join(headers) + " |"
                        sep_row = "| " + " | ".join("---" for _ in headers) + " |"
                        md_rows = [header_row, sep_row]

                        for r in tbl_rows:
                            if isinstance(r, dict):
                                row_cells = []
                                for h in headers:
                                    val = None
                                    if key_map and h in key_map:
                                        val = r.get(key_map[h])
                                    if val is None:
                                        val = r.get(h)
                                    if val is None:
                                        h_norm = h.lower().replace(" ", "_")
                                        for rk, rv in r.items():
                                            if rk.lower().replace(" ", "_") == h_norm:
                                                val = rv
                                                break
                                    cell_str = str(val if val is not None else "").replace("\n", " ").replace("|", "/").strip()
                                    row_cells.append(cell_str)
                                if row_cells and not row_cells[0].strip(" *-_") and any(any(ch.isdigit() for ch in c) for c in row_cells[1:]):
                                    row_cells[0] = "**Total**"
                                md_rows.append("| " + " | ".join(row_cells) + " |")
                            elif isinstance(r, list):
                                row_cells = [str(c if c is not None else "").replace("\n", " ").replace("|", "/").strip() for c in r]
                                # Pad or truncate to match header length exactly
                                if len(row_cells) < len(headers):
                                    row_cells.extend([""] * (len(headers) - len(row_cells)))
                                else:
                                    row_cells = row_cells[:len(headers)]
                                if row_cells and not row_cells[0].strip(" *-_") and any(any(ch.isdigit() for ch in c) for c in row_cells[1:]):
                                    row_cells[0] = "**Total**"
                                md_rows.append("| " + " | ".join(row_cells) + " |")

                        tbl_md = "\n".join(md_rows)
                        if tbl_title:
                            md_tables.append(f"### {tbl_title}\n\n{tbl_md}")
                        else:
                            md_tables.append(tbl_md)

                    if md_tables:
                        combined_body = "\n\n".join(md_tables)
                        raw_text = ("\n\n".join(top_lines) + "\n\n" if top_lines else "") + combined_body
                    else:
                        raw_text = retry_resp.text

                # Update token counts if available
                if hasattr(retry_resp, "usage_metadata") and retry_resp.usage_metadata:
                    u = retry_resp.usage_metadata
                    prompt_tokens += getattr(u, "prompt_token_count", 0) or 0
                    candidates_tokens += getattr(u, "candidates_token_count", 0) or 0
                    total_tokens += getattr(u, "total_token_count", 0) or 0
            except Exception as e:
                logger.error(f"Structured JSON fallback failed: {e}")

        text = raw_text.strip()
        if not text:
            text = "*(Blank / unprinted page - no textual financial content detected)*"

        # Clean enclosing markdown blocks if returned
        if text.startswith("```"):
            lines = text.splitlines()
            if lines[0].startswith("```"):
                lines = lines[1:]
            if lines and lines[-1].strip().startswith("```"):
                lines = lines[:-1]
            text = "\n".join(lines).strip()

        # note numbers only in the Note column; a note's number and title above its own table
        text = normalize_note_tables(text)
        # the page's Source line comes first even when the model left it out
        if source_line and not text.lstrip().startswith("Source:"):
            text = f"{source_line}\n\n{text}"

        return {
            "markdown": text,
            "tokens": total_tokens,
            "prompt_tokens": prompt_tokens,
            "candidates_tokens": candidates_tokens,
            "duration_sec": round(duration_sec, 2),
            "model": model_name
        }

    return execute_with_retry(_call_api, "ocr")
