"""Compact Gemini output formats for extraction and merge, and their decoders."""
import json
from types import SimpleNamespace

import pytest

from app.ocr import config as ocr_config
from app.ocr.services import extraction_service as ext
from app.ocr.services import gemini_json
from app.ocr.services import merge_service as merge
from app.ocr.services.extraction_verifier import extract_markdown_table_grid
from tests.test_ocr import KEY, _seed_page, ocr  # noqa: F401  (fixture)

PAGE = """# Statement of Financial Position
(Amount in BDT)

| not | a table |

| Particulars | Notes | 2024 | 2023 |
| | | Taka | Taka |
|---|---|---|---|
| **Current Assets** | | | |
| Cash and bank balances | 4 | 1,000 | 900 |
| Trade receivables | 5, 6 | (200) | - |
| **Total Current Assets** | | 800 | 900 |

| Item | Consolidated 2024 | Standalone 2024 |
|---|---|---|
| Revenue | 5,000 | 4,000 |
"""


class FakeResponse:
    def __init__(self, payload, prompt=100, cand=20, thoughts=0):
        self.text = payload if isinstance(payload, str) else json.dumps(payload)
        self.candidates = []
        self.usage_metadata = SimpleNamespace(
            prompt_token_count=prompt, candidates_token_count=cand,
            thoughts_token_count=thoughts, total_token_count=prompt + cand + thoughts)


class FakeClient:
    """Stands in for genai.Client; records each call's config."""

    def __init__(self, responder):
        self.calls = []
        self.models = SimpleNamespace(generate_content=self._generate)
        self._responder = responder

    def _generate(self, model, contents, config):
        self.calls.append({"model": model, "contents": contents, "config": config})
        return self._responder(contents, config)


# ---------------------------------------------------------------- shared table numbering
def test_annotator_and_grid_share_numbering():
    annotated = ext.annotate_markdown_tables_with_coordinates(PAGE)
    grid = extract_markdown_table_grid(PAGE)
    # The pipe line without a divider is not a table, so the real tables are 1 and 2.
    assert "| not | a table |" in annotated and "[Table 1: Row" not in annotated.split("| not")[0]
    # Two header lines: both are headers, so the first body row is the section heading.
    assert "[Table 1: Row 1] | **Current Assets**" in annotated
    assert "[Table 1: Row 2] | Cash and bank balances" in annotated
    assert annotated.count("[Table 1 Header]") == 2
    assert grid[(1, 2, 1)] == "Cash and bank balances" and grid[(1, 2, 3)] == "1,000"
    assert "[Table 2: Row 1] | Revenue" in annotated and grid[(2, 1, 3)] == "4,000"


# ---------------------------------------------------------------- extraction decoder
COMPACT = {"t": [
    {"n": 1, "s": "B", "sc": "C", "u": "BDT", "m": 1, "lc": 1, "nc": 2,
     "y": [{"c": 3, "y": 2024}, {"c": 4, "y": 2023}],
     "sec": [{"a": 1, "b": 4, "t": "Current Assets"}],
     "r": [{"r": 2, "c": "Cash and Cash Equivalents", "l": "L"},
           {"r": 3, "c": "=", "l": "L", "q": "C", "f": "label typo"},
           {"r": 4, "c": "Current Assets", "l": "S"}]},
    {"n": 2, "s": "I", "sc": "C", "u": "BDT", "m": 1000, "lc": 1, "nc": 0,
     "y": [{"c": 2, "y": 2024}, {"c": 3, "y": 2024, "sc": "S"}],
     "r": [{"r": 1, "c": "=", "l": "L"}]},
]}


def test_decode_reads_labels_notes_values_from_cells():
    items = ext.decode_compact_extraction(COMPACT, PAGE)
    cash = items[0]
    assert cash["raw_field"] == "Cash and bank balances"
    assert cash["concept"] == {"name": "Cash and Cash Equivalents", "statement": "BALANCE_SHEET",
                               "category": "Current Assets", "level": "LINE_ITEM"}
    assert cash["notes"] == ["4"]
    assert cash["behavior"]["temporal"] == "POINT_IN_TIME"
    assert [(d["year"], d["numeric_value"], d["table_row"], d["table_col"]) for d in cash["data"]] == [
        (2024, 1000.0, 2, 3), (2023, 900.0, 2, 4)]

    receivables = items[1]
    assert receivables["concept"]["name"] == "Trade receivables"  # "=" -> printed label
    assert receivables["notes"] == ["5", "6"]
    assert [d["numeric_value"] for d in receivables["data"]] == [-200.0, 0.0]  # parentheses, nil dash
    assert receivables["behavior"]["qualifier"] == "CURRENT"
    assert receivables["verification_flag"] == "INFERRED_DISCREPANCY"
    assert receivables["flag_reason"] == "label typo"
    assert items[2]["concept"]["level"] == "SUBTOTAL"


def test_decode_splits_scope_columns_and_keeps_multiplier_separate():
    items = ext.decode_compact_extraction(COMPACT, PAGE)
    revenue = [i for i in items if i["raw_field"] == "Revenue"]
    by_scope = {i["scope"]["type"]: i["data"] for i in revenue}
    assert set(by_scope) == {"CONSOLIDATED", "STANDALONE"}
    assert by_scope["CONSOLIDATED"][0]["numeric_value"] == 5000.0  # as printed
    assert by_scope["STANDALONE"][0]["multiplier"] == 1000.0
    assert revenue[0]["behavior"]["temporal"] == "PERIOD_FLOW"


def test_decode_skips_bad_rows_and_tolerates_unknown_codes():
    payload = {"t": [{"n": 1, "s": "?", "lc": 1, "y": [{"c": 3, "y": 2024}],
                      "r": [{"r": 99, "c": "Ghost", "l": "L"}, {"r": 2, "c": "=", "l": "Z", "mt": "??"}]},
                     {"n": 7, "r": [{"r": 1, "c": "=", "l": "L"}]}]}
    items = ext.decode_compact_extraction(payload, PAGE)
    assert len(items) == 1
    assert items[0]["concept"]["statement"] == "UNKNOWN"
    assert items[0]["concept"]["level"] == "LINE_ITEM"
    assert items[0]["behavior"]["metric_type"] == "CURRENCY"


SCHEMA_FORM = {"t": [
    {"n": 1, "s": "B", "st": "Consolidated Statement of Financial Position", "sc": "C",
     "sh": "Consolidated Statement of Financial Position", "u": "BDT", "m": 1, "lc": 1, "nc": 2,
     "y": [{"c": 3, "y": 2024}, {"c": 4, "y": 2023}],
     "sec": [{"a": 1, "b": 4, "t": "Current Assets"}],
     "r": [{"r": 2, "c": "Cash and Cash Equivalents", "l": "L"},
           {"r": 3, "c": "=", "l": "L", "q": "C", "f": "label typo"},
           {"r": 4, "c": "Current Assets", "l": "S"}]},
    {"n": 2, "s": "I", "st": "Consolidated Income Statement", "sc": "C", "sh": "Consolidated Income Statement",
     "u": "BDT", "m": 1000, "lc": 1, "nc": 0,
     "y": [{"c": 2, "y": 2024}, {"c": 3, "y": 2024, "sc": "S"}],
     "r": [{"r": 1, "c": "=", "l": "L"}]},
]}


DATE_PAGE = """| Particulars | Note | 30 June 2023 | July 01, 2021 to June 30, 2022 | 2020-21 |
|---|---|---|---|---|
| Accounts payable | 12 | 1,000 | 900 | 800 |
"""


def test_decode_fills_missing_year_from_date_then_header():
    payload = {"t": [{"n": 1, "s": "B", "lc": 1, "nc": 2,
                      "y": [{"c": 3, "d": "2023-06-30"}, {"c": 4}, {"c": 5}],
                      "r": [{"r": 1, "c": "=", "l": "L"}]}]}
    (item,) = ext.decode_compact_extraction(payload, DATE_PAGE)
    assert [(d["year"], d["date"]) for d in item["data"]] == [(2023, "2023-06-30"), (2022, None), (2021, None)]


def test_schema_is_valid_for_the_sdk():
    from google.genai import types
    sc = types.Schema.model_validate(ext.EXTRACTION_RESPONSE_SCHEMA)
    rows = sc.properties["t"].items.properties["r"].items
    assert rows.required == ["r", "c", "l"] and rows.property_ordering[:3] == ["r", "c", "l"]
    assert set(sc.properties["t"].items.properties["s"].enum) == set(ext.STATEMENT_CODES)
    types.Schema.model_validate(merge.LINK_RESPONSE_SCHEMA)


# ---------------------------------------------------------------- extraction end to end
PAGE_CONS = (PAGE.replace("# Statement of Financial Position", "# Consolidated Statement of Financial Position")
             .replace("| Item |", "## Consolidated Income Statement\n\n| Item |"))


def test_extract_endpoint_with_compact_model_output(ocr, monkeypatch):  # noqa: F811
    _seed_page("COMPACTCO", "2024-25", 9, PAGE_CONS, [])
    fake = FakeClient(lambda contents, config: FakeResponse(SCHEMA_FORM, cand=60, thoughts=15))
    monkeypatch.setattr(ext.genai, "Client", lambda api_key=None: fake)
    monkeypatch.setattr(ocr_config, "GEMINI_THINKING_BUDGET", 0)

    body = ocr.post("/ocr/pipeline/extract", headers=KEY, json={
        "instrument_code": "COMPACTCO", "fiscal_year": "2024-25", "page_numbers": [9]}).json()

    (res,) = body["verification_results"]
    assert res["is_success"] and res["verification_status"] == "VERIFIED"
    assert len(fake.calls) == 1             # breakdowns come from the same single answer (no follow-up call)
    assert res["candidates_tokens"] == 75  # output + thought tokens
    assert fake.calls[0]["config"].thinking_config.thinking_budget == 0
    assert fake.calls[0]["config"].response_schema == ext.EXTRACTION_RESPONSE_SCHEMA
    assert "[Table 1: Row 2]" in fake.calls[0]["contents"]

    cells = ocr.get("/ocr/company/COMPACTCO/extracted-cells", headers=KEY).json()["2024-25_9"]
    cash = [c for c in cells if c["raw_field"] == "Cash and bank balances"]
    assert {c["numeric_value"] for c in cash} == {1000.0, 900.0}
    assert all(c["is_value_match"] for c in cells)


# ---------------------------------------------------------------- thinking config
def test_thinking_config_dropped_when_model_rejects_it(monkeypatch):
    monkeypatch.setattr(ocr_config, "GEMINI_THINKING_BUDGET", 0)
    monkeypatch.setattr(gemini_json, "_thinking_unsupported", set())

    def responder(contents, config):
        if config.thinking_config is not None:
            # What gemini-3.5-flash-lite really answers to thinking_budget=0: the field is not named
            raise ValueError("400 INVALID_ARGUMENT. {'error': {'message': 'Request contains an invalid argument.'}}")
        return FakeResponse({"t": []})

    fake = FakeClient(responder)
    gemini_json.generate_json(fake, "m1", "p", "sys")
    gemini_json.generate_json(fake, "m1", "p", "sys")
    assert [c["config"].thinking_config is not None for c in fake.calls] == [True, False, False]


def test_auto_thinking_sends_no_config(monkeypatch):
    monkeypatch.setattr(ocr_config, "GEMINI_THINKING_BUDGET", None)
    fake = FakeClient(lambda c, cfg: FakeResponse({"t": []}))
    gemini_json.generate_json(fake, "m2", "p", "sys")
    assert fake.calls[0]["config"].thinking_config is None


# ---------------------------------------------------------------- merge: Gemini link judge
def _questions():
    cand = lambda label, ev: {"label": label, "report": "2020-21", "section": "CL", "evidence": ev}  # noqa: E731
    return [
        {"qid": 1, "statement_type": "BALANCE_SHEET",
         "head": {"label": "Accounts Payable", "report": "2019-20", "section": "CL"},
         "candidates": [cand("Creditors and Other Payables", "=2020:1,234,567"), cand("Accrued Expenses", "∅")]},
        {"qid": 2, "statement_type": "BALANCE_SHEET",
         "head": {"label": "Accrued Expenses", "report": "2019-20", "section": "CL"},
         "candidates": [cand("Accrued Expenses and Provisions", "≠2020:+40.0%")]},
        {"qid": 3, "statement_type": "BALANCE_SHEET",
         "head": {"label": "Stock", "report": "2019-20", "section": "CA"},
         "candidates": [cand("Inventories", "∅")]},
    ]


def test_link_batch_input_lists_each_label_once():
    batch_input, labels = merge.build_link_batch_input(_questions())
    assert labels.count("Accrued Expenses") == 1
    assert batch_input["q"][0][:5] == [0, "BALANCE_SHEET", 0, "2019-20", "CL"]      # ids are positions in the batch
    assert [r[0] for r in batch_input["q"]] == [0, 1, 2]
    assert batch_input["q"][0][5][0] == [1, "2020-21", "CL", "=2020:1,234,567"]


def test_link_judge_answer_decoded(monkeypatch):
    # questions are sent sorted by content (Accounts Payable, Accrued Expenses, Stock) and numbered 0..2 in the prompt;
    # labels: 0 Accounts Payable, 1 Creditors and Other Payables, 2 Accrued Expenses,
    #         3 Accrued Expenses and Provisions, 4 Stock, 5 Inventories
    answer = {"t": "LLLLAA", "d": [{"q": 0, "c": 0, "v": "SAME"}, {"q": 1, "c": -1, "v": "DIFFERENT"}]}
    fake = FakeClient(lambda contents, config: FakeResponse(answer, cand=30, thoughts=5))
    monkeypatch.setattr(merge, "get_gemini_client", lambda: fake)
    monkeypatch.setattr(merge._rate_limiter, "wait_sync", lambda: None)
    monkeypatch.setattr(ocr_config, "GEMINI_THINKING_BUDGET", 0)

    answers, usage = merge.judge_link_questions(_questions())

    assert answers[1] == {"choice": 0, "verdict": "SAME", "confidence": 0.95,
                          "type_head": "LIABILITY", "type_choice": "LIABILITY"}
    assert answers[2]["choice"] == -1 and answers[2]["verdict"] == "DIFFERENT"
    assert answers[3]["verdict"] == "NO_ANSWER" and answers[3]["choice"] == -1     # left out by the model: not a "no"
    assert usage["candidates_tokens"] == 35 and usage["batches_count"] == 1 and usage["missing_answers"] == 1
    assert usage["batch_details"][0]["missing_answers"] == 1
    assert fake.calls[0]["config"].temperature == 0.0
    sent = json.loads(fake.calls[0]["contents"].split("\n\n", 1)[1])
    assert set(sent) == {"f", "q"}
    assert fake.calls[0]["config"].thinking_config.thinking_budget == 0
    assert fake.calls[0]["config"].response_schema == merge.LINK_RESPONSE_SCHEMA


@pytest.mark.parametrize("raw,expected", [("SAME", 0.95), ("unsure", 0.5), ("DIFFERENT", 0.0), (95, None), ("x", None)])
def test_link_verdict_scale(raw, expected):
    decoded, _ = merge.decode_link_batch_output({"t": "AA", "d": [[1, 0, raw]]}, ["x", "y"])
    assert (decoded[1]["confidence"] if 1 in decoded else None) == expected


def test_no_candidate_is_always_different():
    decoded, _ = merge.decode_link_batch_output({"t": "AA", "d": [[1, -1, "SAME"]]}, ["x", "y"])
    assert decoded[1]["verdict"] == "DIFFERENT"


def _batch_questions(n, partitions=("A",)):
    out = []
    for i in range(n):
        part = partitions[i % len(partitions)]
        out.append({"qid": 1000 + i, "partition": part, "statement_type": "BALANCE_SHEET", "head_key": f"line {i:03d}",
                    "head": {"label": f"Line {i:03d}", "report": "2019-20", "section": "CL"},
                    "candidates": [{"label": f"Other {i:03d}", "key": f"other {i:03d}", "report": "2020-21", "section": "CL", "evidence": "∅"}]})
    return out


def test_link_prompt_does_not_depend_on_input_order_or_the_global_counter():
    qs = _batch_questions(12, ("A", "B", "C"))
    shuffled = list(reversed(qs))
    renumbered = [dict(q, qid=q["qid"] + 777) for q in qs]
    prompts = []
    for variant in (qs, shuffled, renumbered):
        (batch,) = merge.split_link_batches(variant, 50)
        prompts.append(json.dumps(merge.build_link_batch_input(batch), ensure_ascii=False))
    assert prompts[0] == prompts[1] == prompts[2]


def test_link_batches_are_cut_on_partition_boundaries():
    qs = _batch_questions(30, ("A", "B", "C"))               # 10 questions per partition
    batches = merge.split_link_batches(qs, 25)
    assert [len(b) for b in batches] == [20, 10]
    assert all(len({q["partition"] for q in b}) <= 2 for b in batches)
    assert {q["partition"] for q in batches[1]} == {"C"}
    big = merge.split_link_batches(_batch_questions(60), 25)    # one partition larger than a batch is split in order
    assert [len(b) for b in big] == [25, 25, 10]
    assert [q["head_key"] for b in big for q in b] == sorted(q["head_key"] for q in _batch_questions(60))


def test_scope_quote_from_another_statements_heading_is_rejected():
    md = "CONSOLIDATED CASH FLOW STATEMENT\n\n| A | 1 |\n\nBALANCE SHEET\n\n| B | 2 |"
    ctx = {"has_consolidated": True, "page_titles": []}
    bad = {"t": [{"n": 1, "s": "B", "st": "BALANCE SHEET", "sc": "C", "sh": "CONSOLIDATED CASH FLOW STATEMENT"}]}
    good = {"t": [{"n": 1, "s": "B", "st": "BALANCE SHEET", "sc": "S", "sh": "BALANCE SHEET"}]}
    assert 1 in ext.check_label_evidence(bad, md, ctx)
    assert ext.check_label_evidence(good, md, ctx) == {}


def test_scope_quote_must_come_from_this_page_when_the_page_prints_its_own_heading():
    md = "THE CITY BANK LIMITED\n\nBALANCE SHEET\nAS AT 31 DECEMBER 2013\n\n| A | 1 |"
    ctx = {"has_consolidated": True, "page_titles": ["CONSOLIDATED BALANCE SHEET"]}
    bad = {"t": [{"n": 1, "s": "B", "st": "BALANCE SHEET", "sc": "C", "sh": "CONSOLIDATED BALANCE SHEET"}]}
    assert 1 in ext.check_label_evidence(bad, md, ctx)
    cont = "| A | 1 |"          # a continuation page prints no heading: the report titles are fine
    assert ext.check_label_evidence(bad, cont, ctx) == {}


def test_prompt_names_generic_rows_after_their_parent_heading():
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert "8. GENERIC ROW LABELS" in text
    assert "Net Profit after Tax Attributable to Total" in text
    assert "Balance with Other Banks and Financial Institutions in Bangladesh" in text
    assert 'except as rule 8 requires' in text


def test_prompt_decides_scope_per_table_from_its_own_title():
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert "SCOPE IS DECIDED PER TABLE" in text
    assert "never copy the scope of the" in text and "NEW title" in text
    assert "even when an earlier" in text and "keeps the scope of the table above" in text


def test_prompt_names_subsidiary_rows_after_the_heading_not_the_entity():
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert "ROWS BROKEN DOWN BY SUBSIDIARY" in text and "never write \"<entity> <heading>\"" in text


def test_prompt_starts_a_section_at_a_header_row_that_prints_a_total():
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert "A range starts AT a printed section header row" in text
    assert "including when that header row prints its own total" in text
    assert "Never fold such a row into the previous section" in text
    assert "Give every section its own range" in text and "never the statement's grand-total label" in text
    assert "EVERY row stays inside a range" in text and "never left without a range" in text
    assert "NEAREST printed section header above it, never the side heading" in text
    assert "exactly one range per such row" in text


def test_prompt_treats_less_and_add_rows_as_calculation_steps():
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert 'An operator word printed before a label ("Less:", "Add:"' in text and "never for the value" in text
    assert "CALCULATION-STYLE STATEMENTS (running totals)" in text
    assert "has TWO roles and needs TWO breakdowns" in text
    assert "Never put the header's own parts into the running total" in text
    assert "(100 - 60 = 40)" in text



def test_breakdowns_are_required_and_explained_in_the_one_prompt():
    table = ext.EXTRACTION_RESPONSE_SCHEMA["properties"]["t"]["items"]
    assert "k" in table["required"]
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert "(C = A-B)" in text and "EVERY value column" in text
    assert not hasattr(ext, "request_breakdowns") and not hasattr(ext, "BREAKDOWN_RESPONSE_SCHEMA")


def test_n_is_decided_by_a_mechanical_procedure_with_the_provision_example():
    text = ext.EXTRACTION_SYSTEM_INSTRUCTION
    assert "DECIDE \"n\" WITH THIS PROCEDURE" in text
    assert text.index("STEP 1.") < text.index("STEP 2.") < text.index("STEP 3.")
    assert "\"n\" is\n        EMPTY" in text or "EMPTY: write no \"n\" and stop" in text
    assert "The label never decides" in text
    # the real-bank example: a line bracketed in one year and positive in the other is a plain part
    assert "Provision for diminution in value of investments  (350) / 1,480" in text
    assert "wrong in both columns" in text
    assert "is NEVER in \"n\"" not in text      # the old prohibition wording was replaced


def test_ocr_source_line_and_it_is_never_label_evidence():
    from app.ocr.services import ocr_service
    line = ocr_service.page_source_line({"company_name": "The City Bank Limited", "security_code": "CITYBANK",
                                         "fiscal_year": "2012-13", "page_number": 81})
    assert line == "Source: The City Bank Limited (CITYBANK) | FY 2012-13 | Page 81"
    md = line + "\n\n### Balance Sheet\n\n| Particulars | Note | 2013 |\n|---|---|---|\n| Cash | | 5 |\n"
    ctx = {"has_consolidated": True, "page_titles": []}
    quoted_source = {"t": [{"n": 1, "s": "B", "st": "Balance Sheet", "sc": "S", "sh": "The City Bank Limited (CITYBANK)"}]}
    assert 1 in ext.check_label_evidence(quoted_source, md, ctx)       # the system line is not printed evidence
    printed = {"t": [{"n": 1, "s": "B", "st": "Balance Sheet", "sc": "S", "sh": "Balance Sheet"}]}
    assert ext.check_label_evidence(printed, md, ctx) == {}


def test_row_marked_unidentifiable_is_kept_but_flagged_and_left_out_of_the_csv():
    md = ("| Particulars | Note | 2021 | 2020 |\n|---|---|---|---|\n"
          "| Stationery and stamps | | (1,173) | (1,428) |\n| **Total** | | **(51,242)** | **(82,916)** |\n")
    payload = {"t": [{"n": 1, "s": "N", "st": "x", "sc": "C", "sh": "x", "lc": 1, "nc": 2,
                      "y": [{"c": 3, "y": 2021}, {"c": 4, "y": 2020}], "sec": [{"a": 1, "b": 2, "t": ""}], "k": [],
                      "r": [{"r": 1, "c": "=", "l": "L"}, {"r": 2, "c": "Total", "l": "G", "x": 1}]}]}
    items = ext.decode_compact_extraction(payload, md)
    flags = {i["raw_field"]: i["verification_flag"] for i in items}
    assert flags == {"Stationery and stamps": None, "Total": "FIELD_UNIDENTIFIED"}
    total = next(i for i in items if i["raw_field"] == "Total")
    assert [d["numeric_value"] for d in total["data"]] == [-51242.0, -82916.0]      # the figures are kept
    from app.ocr.services import csv_service
    metrics = [{**i, "field": i["concept"]["name"], "concept_name": i["concept"]["name"],
                "statement_type": i["concept"]["statement"], "scope": "CONSOLIDATED", "page_number": 1,
                "fiscal_year": "2020-21", "id": n} for n, i in enumerate(items)]
    csv = csv_service.build_raw_csv_from_metrics(metrics, "Co", "CO")
    assert "Stationery" in str(csv) and "Total" not in str(csv)


def test_low_confidence_in_the_inferred_heading_marks_the_row_unidentified():
    md = ("| Particulars | Note | 2021 | 2020 |\n|---|---|---|---|\n| Item | | 5 | 4 |\n| Total | | 5 | 4 |\n| Other | | 1 | 1 |\n")
    def run(cf):
        payload = {"t": [{"n": 1, "s": "N", "st": "x", "sc": "C", "sh": "x", "lc": 1, "nc": 2,
                          "y": [{"c": 3, "y": 2021}, {"c": 4, "y": 2020}], "sec": [{"a": 1, "b": 3, "t": "Note 7"}], "k": [],
                          "r": [{"r": 1, "c": "=", "l": "L"}, {"r": 2, "c": "Note 7 total", "l": "G", "cf": cf},
                                {"r": 3, "c": "Note 7 other", "l": "L", "cf": 95}]}]}
        return {i["raw_field"]: i for i in ext.decode_compact_extraction(payload, md)}
    sure, unsure = run(92), run(40)
    assert sure["Total"]["verification_flag"] is None
    assert unsure["Total"]["verification_flag"] == "FIELD_UNIDENTIFIED" and "40%" in unsure["Total"]["flag_reason"]
    assert unsure["Other"]["verification_flag"] is None          # a confident row is kept


def test_user_named_line_enters_the_tables_and_survives_reextraction(tmp_path, monkeypatch):
    from app.ocr import database as db
    monkeypatch.setattr(db, "DB_PATH", tmp_path / "t.db")
    db.init_db()
    item = {"concept": {"name": "Total", "statement": "NOTES_AND_FOOTNOTES", "category": "", "level": "GRAND_TOTAL"},
            "scope": {"type": "CONSOLIDATED", "entity": None, "segment": None},
            "behavior": {"temporal": "PERIOD_FLOW", "action": None, "metric_type": "CURRENCY", "qualifier": None},
            "tag": "FINANCIAL_METRIC", "verification_flag": "FIELD_UNIDENTIFIED", "flag_reason": "x", "raw_field": "Total",
            "notes": [], "data": [{"year": 2021, "date": None, "numeric_value": 5.0, "unit": None, "multiplier": 1.0,
                                   "table_num": 1, "table_row": 8, "table_col": 3}]}
    db.save_extracted_metrics("CO", "2020-21", 322, [item])
    res = db.set_line_override("CO", "2020-21", 322, 1, 8, "Other assets increase/decrease total")
    assert res["updated_metrics"] == 1
    (m,) = db.get_all_metrics_for_company("CO")
    assert m["field"] == "Other assets increase/decrease total" and m["verification_flag"] == "USER_CONFIRMED"
    db.save_extracted_metrics("CO", "2020-21", 322, [item])                    # the page is extracted again
    (m,) = db.get_all_metrics_for_company("CO")
    assert m["field"] == "Other assets increase/decrease total" and m["verification_flag"] == "USER_CONFIRMED"


def test_a_bare_generic_name_is_unidentified_with_the_heading_as_the_suggestion():
    md = ("| Particulars | Note | 2021 | 2020 |\n|---|---|---|---|\n| Item | | 5 | 4 |\n| **Total** | | 5 | 4 |\n| Other | | 1 | 1 |\n")
    payload = {"t": [{"n": 1, "s": "N", "st": "x", "sc": "C", "sh": "x", "lc": 1, "nc": 2,
                      "y": [{"c": 3, "y": 2021}, {"c": 4, "y": 2020}], "sec": [{"a": 1, "b": 3, "t": "Increase of other items"}],
                      "k": [], "r": [{"r": 1, "c": "=", "l": "L"}, {"r": 2, "c": "=", "l": "G"},
                                     {"r": 3, "c": "Increase of other items other", "l": "L"}]}]}
    items = {i["raw_field"]: i for i in ext.decode_compact_extraction(payload, md)}
    assert items["Item"]["verification_flag"] is None
    assert items["Total"]["verification_flag"] == "FIELD_UNIDENTIFIED"
    assert items["Total"]["concept"]["name"] == "Increase of other items Total"      # the suggestion
    assert "Total" in items["Total"]["flag_reason"]
    assert items["Other"]["verification_flag"] is None                                  # a full name is fine


def test_raw_rows_keep_same_named_lines_of_different_headings_apart():
    from app.ocr.services import csv_service
    def metric(cat, row, v):
        return {"field": "Total", "raw_field": "Total", "concept_name": "Total", "category": cat, "statement_type": "NOTES_AND_FOOTNOTES",
                "scope": "CONSOLIDATED", "page_number": 444, "fiscal_year": "2023-24", "aggregation_level": "GRAND_TOTAL",
                "data": [{"year": 2024, "numeric_value": v, "unit": "BDT", "multiplier": 1.0,
                          "table_num": 1, "table_row": row, "table_col": 3}]}
    res = csv_service.build_raw_csv_from_metrics([metric("Other assets", 10, 1.0), metric("Other liabilities", 9, 2.0)], "Co", "CO")
    assert len(res["rows"]) == 2
    assert {r["category"] for r in res["rows"]} == {"Other assets", "Other liabilities"}
    one = csv_service.build_raw_csv_from_metrics([metric("Other assets", 10, 1.0)], "Co", "CO")
    assert len(one["rows"]) == 1


def test_composition_tolerance_uses_each_figures_own_printed_precision():
    md = ("| Particulars | Note | 2018 |\n|---|---|---|\n"
          "| Current Tax (at 25%) | | (290,977,280.58) |\n| **Current tax expenses** | | **(290,977,281)** |\n")
    payload = {"t": [{"n": 1, "lc": 1, "nc": 2, "y": [{"c": 3, "y": 2018}], "k": [{"d": "R", "t": 2, "p": [1]}]}]}
    comps = ext.decode_compositions(payload, md)
    assert [c["status"] for c in comps] == ["MATCH"]          # 0.42 Taka is inside the rounding of the whole-Taka total
    far = md.replace("(290,977,281)", "(290,977,383)")
    assert [c["status"] for c in ext.decode_compositions(payload, far)] == ["MISMATCH"]


# ---------------------------------------------------------------- merge progress: Gemini calls
def test_link_judge_reports_every_call_and_retry(monkeypatch):
    attempts = []

    def responder(contents, config):
        attempts.append(1)
        if len(attempts) == 1:
            raise RuntimeError("503 UNAVAILABLE")
        return FakeResponse({"t": "", "d": []}, cand=3)

    monkeypatch.setattr(merge, "get_gemini_client", lambda: FakeClient(responder))
    monkeypatch.setattr(merge._rate_limiter, "wait_sync", lambda: None)
    monkeypatch.setattr(merge.time, "sleep", lambda s: None)
    monkeypatch.setattr(ocr_config, "GEMINI_THINKING_BUDGET", 0)
    events = []
    merge.judge_link_questions(_batch_questions(5), batch_size=3, progress_callback=events.append)

    assert [(e["stage"], e["call_in_round"]) for e in events] == [
        ("gemini_call_started", 1), ("gemini_retry", 1), ("gemini_call_done", 1),
        ("gemini_call_started", 2), ("gemini_call_done", 2)]
    assert {e["calls_in_round"] for e in events} == {2}
    assert events[1]["retry_attempt"] == 2 and events[1]["retry_in_sec"] == 2
    done = [e for e in events if e["stage"] == "gemini_call_done"]
    assert [e["evaluated_pairs"] for e in done] == [3, 5] and all("call_duration_sec" in e for e in done)
    assert all(e["type"] == "progress" and e["total_pairs"] == 5 for e in events)
