"""Series merge: deterministic classifiers, the linker on synthetic reports, persistence and endpoints."""
import random

import pytest

from app.ocr import database
from app.ocr.services import extraction_service as ext
from app.ocr.services import series_keys as sk
from app.ocr.services import series_linker as sl
from app.ocr.services import series_merge
from tests.test_ocr import KEY, full_results, merge_and_find, ocr  # noqa: F401  (fixture)


# ---------------------------------------------------------------- scope
CONS = "### Consolidated Statement of Financial Position\nAs at 30 June 2024\n\n| a | b |\n|---|---|\n| x | 1 |"
SEPARATE = "### Statement of Financial Position\nAs at 30 June 2024\n\n| a | b |\n|---|---|\n| x | 1 |"
UNTITLED = "| a | b |\n|---|---|\n| x | 1 |"


def test_single_entity_report_is_standalone():
    assert sk.resolve_report_scopes({1: SEPARATE, 2: UNTITLED}, {1: {"CONSOLIDATED"}, 2: {"CONSOLIDATED"}}) == {
        1: "STANDALONE", 2: "STANDALONE"}


def test_consolidated_report_pages():
    scopes = sk.resolve_report_scopes({1: CONS, 2: SEPARATE, 3: UNTITLED, 4: CONS},
                                      {1: {"CONSOLIDATED"}, 2: {"CONSOLIDATED"}, 3: {"STANDALONE"},
                                       4: {"CONSOLIDATED", "STANDALONE"}})
    assert scopes == {1: "CONSOLIDATED", 2: "STANDALONE", 3: "CONSOLIDATED", 4: None}  # 4 prints both scopes


# ---------------------------------------------------------------- value kind, unit, tolerance
@pytest.mark.parametrize("label,values,kind,unit", [
    ("Basic Earnings Per Share", [6.25], "PER_SHARE", "BDT/share"),
    ("Weighted number of shares outstanding", [397569804], "COUNT", "Nos."),
    ("Current tax (at 20.0%)", [20], "PERCENT", "%"),
    ("Income tax @ 27.5%", [5_000_000], "CURRENCY", "BDT"),
    ("Debt equity ratio (times)", [0.45], "RATIO", "Times"),
    ("Accounts Payable", [12_925_122_361], "CURRENCY", "BDT"),
])
def test_value_kind_and_unit(label, values, kind, unit):
    assert sk.value_kind(label, ["BDT"], "CURRENCY", values) == kind
    assert sk.unit_for_kind(kind, ["BDT"]) == unit


@pytest.mark.parametrize("a,b,kind,mult,expected", [
    (6.25, 66.78, "PER_SHARE", 1, False),          # used to pass as a 100-taka "micro variance"
    (6.25, 6.26, "PER_SHARE", 1, True),
    (15.3, 0.153, "PERCENT", 1, True),
    (12_925_122_000, 12_925_122_361, "CURRENCY", 1000, True),   # printed in thousands
    (12_925_122_000, 12_925_222_361, "CURRENCY", 1, False),
])
def test_agree_per_kind(a, b, kind, mult, expected):
    assert sk.agree(a, b, kind, mult, 1) is expected


def test_label_key_ignores_case_scope_and_synonyms():
    assert sk.label_key("Consolidated Sales") == sk.label_key("revenue")
    assert sk.label_key("Property, plant & equipment") == sk.label_key("Property Plant and Equipment")


# ---------------------------------------------------------------- year guard (extraction)
def test_column_without_printed_year_is_the_report_year():
    spec = {"col": 3, "y": 2024}
    assert ext._column_year(spec, [["Particulars", "Note", "Amount"]], report_end=2018) == 2018
    # a ten-year summary keeps its printed old years
    assert ext._column_year({"col": 3, "y": 2008}, [["Particulars", "Note", "2008"]], report_end=2018) == 2008
    assert ext._column_year({"col": 3}, [["Particulars", "Note", "Amount"]], report_end=2018) == 2018
    assert ext._column_year({"col": 3, "y": 2017}, [["Particulars", "Note", "2017"]], report_end=2018) == 2017


# ---------------------------------------------------------------- linker on synthetic reports
def _m(fy, field, values, category="Current Liabilities", stmt="BALANCE_SHEET", page=1, row=1, scope="STANDALONE"):
    return {"id": hash((fy, field, page)) % 10**8, "fiscal_year": fy, "page_number": page, "statement_type": stmt,
            "concept_name": field, "category": category, "aggregation_level": "LINE_ITEM", "scope": scope,
            "entity": None, "segment": None, "temporal_state": "POINT_IN_TIME", "action": None,
            "metric_type": "CURRENCY", "qualifier": None, "raw_field": field, "field": field, "tag": "FINANCIAL_METRIC",
            "data": [{"year": y, "date": None, "numeric_value": v, "unit": "BDT", "multiplier": 1.0,
                      "table_num": 1, "table_row": row, "table_col": 3 + i}
                     for i, (y, v) in enumerate(sorted(values.items(), reverse=True))]}


def _run(metrics, overrides=(), judge=None):
    nodes = sl.build_nodes(metrics, {})
    res = sl.link_company(nodes, list(overrides), judge=judge)
    series, links, review = sl.build_series(nodes, res["links"])
    return nodes, res, series, links, review


class RecordingJudge:
    def __init__(self, choose=-1, conf=0.95):
        self.questions, self.choose, self.conf = [], choose, conf

    def __call__(self, questions):
        self.questions.extend(questions)
        return {q["qid"]: {"choice": self.choose, "confidence": self.conf, "type_head": "", "type_choice": ""}
                for q in questions}


def _names(series):
    return [[n["fields"][0] for n in s["nodes"]] for s in series]


def test_rename_linked_by_comparative_value_without_gemini():
    judge = RecordingJudge()
    _, _, series, links, _ = _run([
        _m("2019-20", "Accounts Payable", {2020: 1_234_567, 2019: 1_000_001}),
        _m("2020-21", "Creditors and Other Payables", {2021: 1_500_003, 2020: 1_234_567}),
    ], judge=judge)
    assert [l["rule"] for l in links] == ["VALUE_UNIQUE"]
    assert _names(series) == [["Accounts Payable", "Creditors and Other Payables"]]
    assert judge.questions == []


def test_accounting_identity_is_never_cross_linked():
    m = []
    for fy, y in (("2019-20", 2020), ("2020-21", 2021)):
        m += [_m(fy, "Total Assets", {y: 9_876_543, y - 1: 8_765_432}, category="Assets", row=10),
              _m(fy, "Total Equity and Liabilities", {y: 9_876_543, y - 1: 8_765_432}, category="Equity and Liabilities", row=30)]
    _, _, series, links, _ = _run(m)
    assert sorted(_names(series)) == [["Total Assets", "Total Assets"],
                                      ["Total Equity and Liabilities", "Total Equity and Liabilities"]]
    assert {l["rule"] for l in links} == {"NAME_EXACT"}


def test_coincidental_equal_value_goes_to_gemini():
    judge = RecordingJudge(choose=-1, conf=0.9)
    _, res, series, links, _ = _run([
        _m("2019-20", "Advance Tax Paid", {2020: 160_281_671}, category="Tax"),
        _m("2020-21", "Minimum Tax", {2021: 170_000_000, 2020: 160_281_671}, category="Other"),
    ], judge=judge)
    assert links == [] and len(series) == 2
    (q,) = judge.questions
    assert q["head"]["label"] == "Advance Tax Paid" and q["candidates"][0]["evidence"].startswith("=2020")


def test_similar_names_with_unique_value_link_without_gemini():
    judge = RecordingJudge()
    _, _, _, links, _ = _run([_m("2019-20", "Advance Tax Paid", {2020: 160_281_671}, category="Tax"),
                              _m("2020-21", "Tax Paid in Advance", {2021: 1_700_001, 2020: 160_281_671}, category="Other")],
                             judge=judge)
    assert [l["rule"] for l in links] == ["VALUE_UNIQUE"] and judge.questions == []


def test_gemini_is_asked_again_on_every_run():
    metrics = [_m("2019-20", "Advance Income Tax", {2020: 160_281_671}, category="Tax"),
               _m("2020-21", "Tax Deducted at Source", {2021: 170_000_000, 2020: 160_281_671}, category="Other")]
    first = RecordingJudge(choose=0, conf=0.92)
    _, res, _, links, _ = _run(metrics, judge=first)
    assert [l["rule"] for l in links] == ["GEMINI"] and len(first.questions) == 1
    assert "new_judgements" not in res
    again = RecordingJudge(choose=-1)
    _, _, _, links2, _ = _run(metrics, judge=again)
    assert links2 == [] and len(again.questions) == 1     # nothing is remembered: the new answer decides


def test_unchosen_candidates_are_not_remembered():
    head = _m("2019-20", "Advance Income Tax", {2020: 160_281_671}, category="Tax")
    cand_a = _m("2020-21", "Tax Deducted at Source", {2021: 170_000_000, 2020: 160_281_671}, category="Other")
    cand_b = _m("2020-21", "Prepaid Tax", {2021: 5_000_000, 2020: 160_281_671}, category="Other", row=2)
    first = RecordingJudge(choose=0)
    _, _, _, links, _ = _run([head, cand_a, cand_b], judge=first)
    assert len(links) == 1
    second = RecordingJudge(choose=0)
    _, _, _, links2, _ = _run([head, cand_b], judge=second)       # the earlier winner is gone
    assert len(second.questions) <= 1 and all(l["to_label"] == "prepaid tax" for l in links2)


def test_split_is_detected_not_linked():
    _, res, series, links, _ = _run([
        _m("2019-20", "Trade and Other Payables", {2020: 3_000_000}),
        _m("2020-21", "Trade Payables", {2021: 2_100_000, 2020: 2_000_000}),
        _m("2020-21", "Other Payables", {2021: 1_100_000, 2020: 1_000_000}),
    ])
    assert links == []
    assert [i["kind"] for i in res["review_items"]] == ["SPLIT_DETECTED"]


def test_missing_report_and_skipped_year_are_gap_linked():
    _, _, series, links, _ = _run([
        _m("2014-15", "Security Deposits", {2015: 10_000_000}),
        _m("2016-17", "Security Deposits", {2017: 12_000_000}),            # no 2015-16 report at all
        _m("2016-17", "Retention Money", {2017: 5_000_000}),
        _m("2017-18", "Retention Money", {2018: 6_000_000}),               # Security Deposits not printed
        _m("2018-19", "Security Deposits", {2019: 14_000_000}),
        _m("2018-19", "Retention Money", {2019: 7_000_000}),
    ])
    names = sorted(_names(series))
    assert names == [["Retention Money"] * 3, ["Security Deposits"] * 3]
    assert "GAP_NAME" in {l["rule"] for l in links}


def test_chain_whose_labels_coexist_is_broken():
    nodes = sl.build_nodes([
        _m("2019-20", "Alpha Reserve", {2020: 1}), _m("2019-20", "Beta Reserve", {2020: 2}, row=2),
        _m("2020-21", "Gamma Reserve", {2021: 3}), _m("2021-22", "Beta Reserve", {2022: 4}),
    ], {})
    by = {(n["report"], n["fields"][0]): n for n in nodes}
    mk = lambda a, b, rule, conf: sl._link(a, b, rule, "auto", conf, sl.evidence(a, b))  # noqa: E731
    links = [mk(by[("2019-20", "Alpha Reserve")], by[("2020-21", "Gamma Reserve")], "VALUE_UNIQUE", 0.95),
             mk(by[("2020-21", "Gamma Reserve")], by[("2021-22", "Beta Reserve")], "GEMINI", 0.91)]
    series, kept, review = sl.build_series(nodes, links)
    assert [l["rule"] for l in kept] == ["VALUE_UNIQUE"]
    assert [r["kind"] for r in review] == ["SERIES_INCONSISTENT"]


def test_linking_is_deterministic():
    metrics = [_m(fy, name, {y: v * k, y - 1: v * (k - 1)}, row=r)
               for fy, y, k in (("2019-20", 2020, 3), ("2020-21", 2021, 4), ("2021-22", 2022, 5))
               for r, (name, v) in enumerate((("Cash", 1_000_003), ("Inventories", 2_000_011), ("Payables", 3_000_017)), 1)]
    first = _run(metrics)[3]
    shuffled = metrics[:]
    random.Random(7).shuffle(shuffled)
    assert [(l["from_node"], l["to_node"]) for l in _run(shuffled)[3]] == [(l["from_node"], l["to_node"]) for l in first]


def _override(kind, a, b=None, **kw):
    return {"id": kw.pop("id", 1), "kind": kind, "statement_type": "BALANCE_SHEET", "scope": "STANDALONE",
            "entity_key": "", "segment_key": "", "a_label_key": a, "b_label_key": b, "a_report": None,
            "b_report": None, "new_label": kw.pop("new_label", None), "created_at": "2026-01-01"}


def test_user_overrides_split_link_and_rename():
    metrics = [_m("2019-20", "Accounts Payable", {2020: 1_234_567}),
               _m("2020-21", "Creditors and Other Payables", {2021: 1_500_003, 2020: 1_234_567}),
               _m("2020-21", "Stock", {2021: 99})]
    assert _run(metrics, [_override("FORCE_SPLIT", "Accounts Payable", "Creditors and Other Payables")])[3] == []
    _, _, _, links, _ = _run([_m("2019-20", "Stores", {2020: 5}), _m("2020-21", "Spares", {2021: 9})],
                             [_override("FORCE_LINK", "Stores", "Spares")])
    assert [l["rule"] for l in links] == ["ALIAS_EQUIV"]
    nodes, _, series, _, _ = _run(metrics)
    out = sl.series_to_rows(series, list(range(1, len(series) + 1)),
                            [_override("RENAME", "Accounts Payable", new_label="Trade Payables")], [])
    assert "Trade Payables" in [r["field"] for r in out["rows"]]


def test_rows_resolve_latest_report_restatements_and_merged_from():
    _, _, series, _, _ = _run([
        _m("2019-20", "Accounts Payable", {2020: 1_234_567, 2019: 1_000_001}),
        _m("2020-21", "Accounts payable", {2021: 1_500_003, 2020: 1_300_000}),   # 2020 restated
    ])
    (row,) = sl.series_to_rows(series, [42], [], [])["rows"]
    assert row["series_id"] == 42 and row["values"]["2020"] == 1_300_000 and row["stating_years"]["2020"] == 2021
    assert row["restatement_variances"]["2020"]["original_value"] == 1_234_567
    assert set(row["merged_from"]) == {"Accounts Payable", "Accounts payable"}
    assert row["unit"] == "BDT" and [c["report_year"] for c in row["constituent_details"]] == ["2019-20", "2020-21"]


def test_line_names_put_consolidated_first_leave_standalone_plain_and_append_a_distinct_entity():
    def rows(metrics):
        _, _, series, _, _ = _run(metrics)
        return sl.series_to_rows(series, list(range(1, len(series) + 1)), [], [])["rows"]

    both = rows([_m("2020-21", "Total assets", {2021: 5}, scope="CONSOLIDATED"),
                 _m("2020-21", "Total assets", {2021: 4}, scope="STANDALONE")])
    assert sorted(r["field"] for r in both) == ["Total assets", "Total assets"]
    assert {r["base_field"] for r in both} == {"Total assets"} and {r["merged_from"][0] for r in both} == {"Total assets"}
    assert {r["scope_label"] for r in both} == {"Consolidated", "Standalone"}
    other = _m("2020-21", "Total assets", {2021: 3}, scope="STANDALONE")
    other["entity"] = "City Brokerage Limited"
    names = [r["field"] for r in rows([_m("2020-21", "Total assets", {2021: 5}, scope="CONSOLIDATED"),
                                       _m("2020-21", "Total assets", {2021: 4}, scope="STANDALONE"), other])]
    assert "Total assets - City Brokerage Limited" in names
    assert sl.display_name("Total assets", "CONSOLIDATED", "City Brokerage Limited") == "Total assets - City Brokerage Limited"
    assert sl.display_name("Loans", "STANDALONE", None) == "Loans"


# ---------------------------------------------------------------- persistence and endpoints
MD = "### Statement of Financial Position\n\n| Particulars | Note | 2021 | 2020 |\n|---|---|---|---|\n| x | | 1 | 2 |"


def _seed(code, fy, page, items):
    database.upsert_page(None, code, fy, page, "", is_selected=True)
    database.update_page_ocr(code, fy, page, MD)
    database.save_extracted_metrics(code, fy, page, items)


def _item(field, values, scope="STANDALONE"):
    return {"concept": {"name": field, "statement": "BALANCE_SHEET", "category": "Current Liabilities"},
            "scope": {"type": scope}, "raw_field": field,
            "data": [{"year": y, "numeric_value": v, "unit": "BDT", "multiplier": 1.0, "table_num": 1,
                      "table_row": 1, "table_col": 3 + i} for i, (y, v) in enumerate(values)]}


@pytest.fixture
def renamed(ocr):  # noqa: F811
    _seed("RENCO", "2019-20", 5, [_item("Accounts Payable", [(2020, 1_234_567), (2019, 1_000_001)])])
    _seed("RENCO", "2020-21", 6, [_item("Creditors and Other Payables", [(2021, 1_500_003), (2020, 1_234_567)])])
    return ocr


def test_merge_endpoint_builds_series_and_keeps_the_models_scope(renamed):
    body = renamed.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "RENCO"}).json()
    (row,) = body["merged_csv_data"]["rows"]
    assert row["scope"] == "STANDALONE"           # as labelled at extraction; the merge does not rewrite scope
    assert row["merged_from"] == ["Creditors and Other Payables", "Accounts Payable"]
    assert [l["rule"] for l in row["link_summary"]] == ["VALUE_UNIQUE"]
    sid = row["series_id"]

    results = full_results(renamed, "RENCO")
    assert results["merged_rows"][0]["series_id"] == sid and results["has_merged_csv"]
    series = renamed.get("/ocr/company/RENCO/series", headers=KEY).json()["series"]
    assert [s["id"] for s in series] == [sid] and len(series[0]["members"]) == 2
    ev = renamed.get(f"/ocr/company/RENCO/series/{sid}/evidence", headers=KEY).json()
    assert ev["links"][0]["evidence"]["class"] == "AGREE"

    cell = renamed.get("/ocr/company/RENCO/cell-details", headers=KEY,
                       params={"year": 2020, "series_id": sid}).json()
    assert {r["report_year"] for r in cell["cell_records"]} == {"2019-20", "2020-21"}

    # A second merge keeps the series id
    again = renamed.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "RENCO"}).json()
    assert again["merged_csv_data"]["rows"][0]["series_id"] == sid


def test_override_survives_reextraction_and_marks_merge_stale(renamed):
    renamed.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "RENCO"})
    r = renamed.post("/ocr/company/RENCO/merge/override", headers=KEY, json={
        "kind": "FORCE_SPLIT", "statement_type": "BALANCE_SHEET", "scope": "STANDALONE",
        "a_label": "Accounts Payable", "b_label": "Creditors and Other Payables"})
    assert r.status_code == 201
    assert full_results(renamed, "RENCO")["merged_stale"] is True

    # re-extraction: new metric ids, same lines
    _seed("RENCO", "2020-21", 6, [_item("Creditors and Other Payables", [(2021, 1_500_003), (2020, 1_234_567)])])
    rows = renamed.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "RENCO"}).json()["merged_csv_data"]["rows"]
    assert sorted(r["field"] for r in rows) == ["Accounts Payable", "Creditors and Other Payables"]

    oid = r.json()["id"]
    assert renamed.delete(f"/ocr/company/RENCO/merge/override/{oid}", headers=KEY).status_code == 200
    assert renamed.get("/ocr/company/RENCO/merge/overrides", headers=KEY).json()["overrides"] == []


def test_override_validation(renamed):
    base = {"statement_type": "BALANCE_SHEET", "scope": "STANDALONE", "a_label": "Accounts Payable"}
    assert renamed.post("/ocr/company/RENCO/merge/override", headers=KEY, json={**base, "kind": "FORCE_LINK"}).status_code == 422
    r = renamed.post("/ocr/company/RENCO/merge/override", headers=KEY, json={
        **base, "kind": "FORCE_LINK", "b_label": "Other", "a_report": "2020-21", "b_report": "2020-21"})
    assert r.status_code == 409


def test_review_item_resolution_creates_override(ocr, monkeypatch):  # noqa: F811
    _seed("AMBCO", "2019-20", 5, [_item("Advance Income Tax", [(2020, 160_281_671)])])
    _seed("AMBCO", "2020-21", 6, [{**_item("Tax Deducted at Source", [(2021, 170_000_000), (2020, 160_281_671)]),
                                   "concept": {"name": "Tax Deducted at Source", "statement": "BALANCE_SHEET", "category": "Other"}}])
    judge = RecordingJudge(choose=0, conf=0.70)   # below the 0.80 bar for a unique value match -> review item
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: judge)
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "AMBCO"})
    items = ocr.get("/ocr/company/AMBCO/merge/review", headers=KEY).json()["items"]
    (item,) = [i for i in items if i["kind"] == "AMBIGUOUS_LINK"]
    r = ocr.post(f"/ocr/company/AMBCO/merge/review/{item['id']}/resolve", headers=KEY, json={"action": "accept"}).json()
    assert r["status"] == "resolved" and r["override_id"]
    rows = ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "AMBCO"}).json()["merged_csv_data"]["rows"]
    assert len(rows) == 1 and rows[0]["link_summary"][0]["rule"] == "USER"
    assert len(judge.questions) == 1               # the second merge did not ask again


# ---------------------------------------------------------------- A: scope / statement evidence
NOTE_PAGE = ("2016 FINANCIAL STATEMENTS\nTHE CITY BANK LIMITED\n\n"
             "| Particulars | 2016 | 2015 |\n|---|---|---|\n"
             "| **Increase/(decrease) of other liabilities - The City Bank Limited** | | |\n"
             "| Branch adjustment account | 0 | (7,931,141) |\n")
CTX_CONS = {"fiscal_year": "2015-16", "has_consolidated": True,
            "page_titles": [{"page": 1, "title": "Consolidated Balance Sheet"}]}
BANK_NOTE = "Increase/(decrease) of other liabilities - The City Bank Limited"


def _table(**kw):
    return {"t": [{"n": 1, "lc": 1, "nc": 0, "y": [{"c": 2, "y": 2016}], "sec": [], "r": [], **kw}]}


@pytest.mark.parametrize("table,ctx,ok", [
    (dict(s="N", st="Increase/(decrease) of other liabilities", sc="S", sh=BANK_NOTE), CTX_CONS, True),
    (dict(s="C", st="Increase/(decrease) of other liabilities", sc="S", sh=BANK_NOTE), CTX_CONS, False),  # a note is not CASH_FLOW
    (dict(s="N", st="Increase/(decrease) of other liabilities", sc="C", sh=BANK_NOTE), CTX_CONS, False),  # names the bank: not C
    (dict(s="N", st="Increase/(decrease) of other liabilities", sc="C", sh="THE CITY BANK LIMITED"),
     {**CTX_CONS, "has_consolidated": False}, False),                                                  # no consolidated statements
    (dict(s="B", st="Consolidated Balance Sheet", sc="S", sh="Some heading that is not printed"), CTX_CONS, False),
    (dict(s="B", st="Consolidated Balance Sheet", sc="S", sh="THE CITY BANK LIMITED"), CTX_CONS, True),  # title from the report
])
def test_label_evidence_check(table, ctx, ok):
    failures = ext.check_label_evidence(_table(**table), NOTE_PAGE, ctx)
    assert (failures == {}) is ok, failures


def test_section_scope_must_match_its_heading():
    payload = _table(s="N", st="Increase/(decrease) of other liabilities", sc="S", sh="THE CITY BANK LIMITED",
                     sec=[{"a": 1, "b": 2, "t": BANK_NOTE, "sc": "C"}])
    assert "section" in ext.check_label_evidence(payload, NOTE_PAGE, CTX_CONS)[1]


class _Seq:
    """genai.Client stand-in answering a fixed sequence of payloads."""

    def __init__(self, answers):
        from types import SimpleNamespace
        self.calls, self._answers, self._ns = [], list(answers), SimpleNamespace
        self.models = SimpleNamespace(generate_content=self._gen)

    def _gen(self, model, contents, config):
        import json
        self.calls.append(contents)
        payload = self._answers[min(len(self.calls), len(self._answers)) - 1]
        return self._ns(text=json.dumps(payload), candidates=[],
                        usage_metadata=self._ns(prompt_token_count=10, candidates_token_count=5,
                                                thoughts_token_count=0, total_token_count=15))


def _note_answer(scope, stmt="N"):
    return {"t": [{"n": 1, "s": stmt, "st": "Increase/(decrease) of other liabilities", "sc": scope, "sh": BANK_NOTE,
                   "lc": 1, "nc": 0, "y": [{"c": 2, "y": 2016}, {"c": 3, "y": 2015}],
                   "sec": [{"a": 1, "b": 2, "t": "Increase/(decrease) of other liabilities"}],
                   "r": [{"r": 2, "c": "=", "l": "L"}]}]}


def test_prompt_carries_the_report_context():
    ctx = {**CTX_CONS, "previous_fiscal_year": "2014-15",
           "previous_labels": [{"heading": "Other Liabilities", "statement": "NOTES_AND_FOOTNOTES", "scope": "STANDALONE"}]}
    prompt = ext.build_extraction_prompt(NOTE_PAGE, report_context=ctx)
    assert "contains CONSOLIDATED statements" in prompt and '"Consolidated Balance Sheet"' in prompt
    assert '"Other Liabilities" -> s=NOTES_AND_FOOTNOTES, sc=STANDALONE' in prompt
    single = ext.build_extraction_prompt(NOTE_PAGE, report_context={**CTX_CONS, "has_consolidated": False})
    assert "NO consolidated statements" in single


def test_report_label_context_from_the_database(ocr):  # noqa: F811
    database.upsert_page(None, "CTXCO", "2014-15", 1, "", is_selected=True)
    database.update_page_ocr("CTXCO", "2014-15", 1, "### Statement of Financial Position\n\n| a | 2015 |\n|---|---|\n| x | 1 |")
    database.save_extracted_metrics("CTXCO", "2014-15", 1, [_item("Cash", [(2015, 5)])])
    database.upsert_page(None, "CTXCO", "2015-16", 2, "", is_selected=True)
    database.update_page_ocr("CTXCO", "2015-16", 2, "### Consolidated Statement of Financial Position\n\n| a | 2016 |\n|---|---|\n| x | 1 |")
    ctx = database.get_report_label_context("CTXCO", "2015-16")
    assert ctx["has_consolidated"] and "Consolidated" in ctx["page_titles"][0]["title"]
    assert ctx["previous_fiscal_year"] == "2014-15"
    assert ctx["previous_labels"] == [{"heading": "Current Liabilities", "statement": "BALANCE_SHEET", "scope": "STANDALONE"}]
    assert database.get_report_label_context("CTXCO", "2014-15")["has_consolidated"] is False


# ---------------------------------------------------------------- C: nearest first, then wider
YEARS = [f"{y}-{(y + 1) % 100:02d}" for y in range(2012, 2024)]


def _reports_with_filler(years, line_years, name="Branch adjustment account"):
    """A filler line in every report (so every report exists in company order), `name` only in line_years."""
    m = []
    for i, fy in enumerate(years):
        y = int(fy[:4]) + 1
        m.append(_m(fy, "Filler Line", {y: 1000 + i}, category="Other Liabilities", row=1))
        if fy in line_years:
            m.append(_m(fy, name, {y: 777_000 + i}, category="Other Liabilities", row=2))
    return m


def _branch_links(links):
    return {(l["from_report"], l["to_report"]): (l["rule"], l["gap"]) for l in links if "branch" in l["to_label"]}


def test_gap_is_counted_in_company_order_not_partition_order():
    _, _, series, links, _ = _run(_reports_with_filler(YEARS, {"2012-13", "2023-24"}))
    assert _branch_links(links) == {}                       # 10 reports apart: never joined


def test_widening_one_gap_then_long_gap_with_review_item():
    _, res, _, links, _ = _run(_reports_with_filler(YEARS, {"2012-13", "2014-15", "2017-18"}))
    assert _branch_links(links) == {("2012-13", "2014-15"): ("GAP_NAME", 1), ("2014-15", "2017-18"): ("LONG_GAP", 2)}
    assert [i["payload"]["reports_skipped"] for i in res["review_items"] if i["kind"] == "LONG_GAP"] == [2]


def test_beyond_the_gap_limit_nothing_is_linked():
    metrics = _reports_with_filler(YEARS, {"2012-13", "2017-18"})        # 4 reports skipped
    assert _branch_links(_run(metrics)[3]) == {}
    wide = sl.link_company(sl.build_nodes(metrics, {}), [], {}, max_gap=4)["links"]
    assert _branch_links(wide) == {("2012-13", "2017-18"): ("LONG_GAP", 4)}


def test_closer_report_wins_over_a_wider_one():
    _, _, _, links, _ = _run(_reports_with_filler(YEARS[:4], {"2012-13", "2014-15", "2015-16"}))
    assert _branch_links(links) == {("2012-13", "2014-15"): ("GAP_NAME", 1), ("2014-15", "2015-16"): ("NAME_EXACT", 0)}


def test_same_label_under_assets_and_liabilities_headings_stays_apart():
    def m(fy, cat, v):
        x = _m(fy, "Branch adjustment account", {int(fy[:4]) + 1: v}, category=cat, stmt="NOTES_AND_FOOTNOTES")
        return x
    metrics = [m("2012-13", "Increase/(decrease) of other assets - The City Bank Limited", 5),
               m("2013-14", "Increase/ (decrease) of other liabilities - The City Bank Limited", 6),
               m("2014-15", "Increase/ (decrease) of other liabilities - The City Bank Limited", 7)]
    assert sk.parent_side("(Increase) / decrease of other assets - City Bank PLC", "OTHER") == "A"
    assert sk.parent_side("Current Liabilities", "CL") == ""
    nodes, _, series, links, _ = _run(metrics)
    assert len(series) == 2 and [(l["from_report"], l["to_report"]) for l in links] == [("2013-14", "2014-15")]


def test_zero_valued_line_is_not_a_split_or_merge_part_and_still_links():
    metrics = [_m("2020-21", "Balance with Bangladesh Bank", {2021: 23_142_429_928}, category="PROPERTY AND ASSETS"),
               _m("2020-21", "Money at Call and Short Notice", {2021: 0}, category="PROPERTY AND ASSETS", row=2),
               _m("2021-22", "Balance with central bank", {2022: 24_000_000_000, 2021: 23_142_429_928}, category="PROPERTY AND ASSETS"),
               _m("2021-22", "Money at call on short notice", {2022: 0, 2021: 0}, category="PROPERTY AND ASSETS", row=2)]
    judge = RecordingJudge(choose=0, conf=0.97)
    _, res, _, links, _ = _run(metrics, judge=judge)
    assert not [i for i in res["review_items"] if i["kind"] in ("SPLIT_DETECTED", "MERGE_DETECTED")]
    assert any("money at call" in l["from_label"] and "money at call" in l["to_label"] for l in links)


def test_series_that_stop_and_restart_with_a_new_name_are_joined_by_one_batched_gemini_pass():
    def ni(fy, name, vals, row=1):
        return _m(fy, name, vals, category="Net profit after tax attributable to:", stmt="INCOME_STATEMENT", row=row)
    metrics = [ni("2017-18", "Net profit after tax attributable to total", {2018: 100, 2017: 90}),
               ni("2018-19", "Dividend income", {2019: 120, 2018: 100}, row=9),    # an unrelated line
               ni("2019-20", "Dividend income", {2020: 130, 2019: 120}, row=9),
               ni("2020-21", "Total Net Profit After Tax Attributable", {2021: 150, 2020: 130}),
               ni("2021-22", "Total Net Profit After Tax Attributable", {2022: 160, 2021: 150})]
    judge = RecordingJudge(choose=0, conf=0.97)
    _, res, series, links, _ = _run(metrics, judge=judge)
    joins = [l for l in links if l["rule"] == "GEMINI_JOIN"]
    assert len(joins) == 1 and joins[0]["gap"] == 2 and joins[0]["from_report"] == "2017-18"
    assert any(i["kind"] == "LONG_GAP" and i["payload"].get("decided_by") == "gemini" for i in res["review_items"])
    # nothing is asked for generic one-word labels
    _, _, _, links2, _ = _run([_m("2013-14", "Subtotal", {2014: 1}), _m("2021-22", "Subtotal", {2022: 1})], judge=RecordingJudge())
    assert not [l for l in links2 if l["rule"] == "GEMINI_JOIN"]


# ---------------------------------------------------------------- breakdowns (cell compositions)
COMP_MD = """| Particulars | Note | 2022 | 2021 |
|---|---|---|---|
| In Bangladesh | | 29,219 | 22,015 |
| Outside Bangladesh | | 3,914 | 1,144 |
| Total balance with other banks | | 33,134 | 23,159 |
| Gross | | 100 | 90 |
| Less: Depreciation | | 30 | 20 |
| Net | | 70 | 75 |

| Particulars | Conventional | Islamic | Total |
|---|---|---|---|
| Income | 10 | 5 | 15 |
| Expense | 4 | 1 | 7 |
"""

COMP_PAYLOAD = {"t": [
    {"n": 1, "s": "B", "st": "x", "sc": "S", "sh": "x", "lc": 1, "nc": 2,
     "y": [{"c": 3, "y": 2022}, {"c": 4, "y": 2021}], "sec": [{"a": 1, "b": 6, "t": "Assets"}],
     "k": [{"d": "R", "t": 3, "p": [1, 2]}, {"d": "R", "t": 6, "p": [4, 5], "n": [5]}],
     "r": [{"r": i, "c": "=", "l": "L"} for i in range(1, 7)]},
    {"n": 2, "s": "N", "st": "x", "sc": "S", "sh": "x", "lc": 1, "nc": 0,
     "y": [{"c": 2, "y": 2022, "g": "Conventional"}, {"c": 3, "y": 2022, "g": "Islamic"}, {"c": 4, "y": 2022}],
     "sec": [{"a": 1, "b": 2, "t": "Income"}],
     "k": [{"d": "C", "t": 4, "p": [2, 3]}],
     "r": [{"r": 1, "c": "=", "l": "L"}, {"r": 2, "c": "=", "l": "L"}]},
]}


def test_breakdowns_are_decoded_per_cell_and_checked_against_the_printed_numbers():
    comps = ext.decode_compositions(COMP_PAYLOAD, COMP_MD)
    by = {(c["table_num"], c["total"]["row"], c["total"]["col"]): c for c in comps}
    total_2021 = by[(1, 3, 4)]
    assert total_2021["direction"] == "ROWS" and total_2021["status"] == "MATCH"
    assert [(p["row"], p["col"], p["value"]) for p in total_2021["parts"]] == [(1, 4, 22015.0), (2, 4, 1144.0)]
    assert by[(1, 6, 3)]["status"] == "MATCH" and by[(1, 6, 3)]["parts"][1]["sign"] == -1   # 100 - 30 = 70
    assert by[(1, 6, 4)]["status"] == "MISMATCH" and by[(1, 6, 4)]["difference"] == -5.0   # 90 - 20 != 75
    assert by[(2, 1, 4)]["direction"] == "COLUMNS" and by[(2, 1, 4)]["status"] == "MATCH"  # 10 + 5 = 15
    assert by[(2, 2, 4)]["status"] == "MISMATCH"                                            # 4 + 1 != 7
    assert by[(1, 3, 3)]["status"] == "MATCH"          # 29,219 + 3,914 = 33,133: within printed rounding of 33,134
    assert len(comps) == 6


def test_breakdowns_are_saved_per_page_resolved_to_metrics_and_served(ocr):  # noqa: F811
    database.upsert_page(None, "BRKCO", "2021-22", 7, "", is_selected=True)
    database.update_page_ocr("BRKCO", "2021-22", 7, COMP_MD)
    items = ext.decode_compact_extraction(COMP_PAYLOAD, COMP_MD, report_end=2022)
    database.save_extracted_metrics("BRKCO", "2021-22", 7, items)
    assert database.save_cell_compositions("BRKCO", "2021-22", 7, ext.decode_compositions(COMP_PAYLOAD, COMP_MD)) == 6
    body = ocr.get("/ocr/company/BRKCO/year/2021-22/compositions?page_number=7", headers=KEY).json()
    assert body["count"] == 6 and body["mismatch_count"] == 2
    c = next(x for x in body["compositions"] if (x["table_num"], x["total"]["row"], x["total"]["col"]) == (1, 3, 4))
    assert c["total"]["raw_field"] == "Total balance with other banks" and c["total"]["year"] == 2021
    assert [p["raw_field"] for p in c["parts"]] == ["In Bangladesh", "Outside Bangladesh"]
    links = database.get_cell_composition_links("BRKCO", "2021-22", 7, 1, 1, 4)
    assert links["breakdown"] is None and [x["total"]["row"] for x in links["part_of"]] == [3]
    database.save_cell_compositions("BRKCO", "2021-22", 7, [])               # re-extraction replaces the page
    assert database.get_cell_compositions("BRKCO", "2021-22", 7) == []


def test_merged_rows_carry_the_breakdown_as_series_ids():
    items = ext.decode_compact_extraction(COMP_PAYLOAD, COMP_MD, report_end=2022)
    metrics = []
    for i, it in enumerate(items):
        metrics.append({"id": i, "fiscal_year": "2021-22", "page_number": 7, "statement_type": it["concept"]["statement"],
                        "concept_name": it["concept"]["name"], "category": it["concept"]["category"],
                        "aggregation_level": "LINE_ITEM", "scope": it["scope"]["type"], "entity": None,
                        "segment": it["scope"]["segment"], "temporal_state": "POINT_IN_TIME", "action": None,
                        "metric_type": "CURRENCY", "qualifier": None, "raw_field": it["raw_field"],
                        "field": it["concept"]["name"], "tag": "FINANCIAL_METRIC", "data": it["data"]})
    comps = [{**c, "fiscal_year": "2021-22", "page_number": 7} for c in ext.decode_compositions(COMP_PAYLOAD, COMP_MD)]
    nodes = sl.build_nodes(metrics, {})
    series, _, _ = sl.build_series(nodes, [])
    ids = list(range(1, len(series) + 1))
    rows = {r["field"]: r for r in sl.series_to_rows(series, ids, [], [], compositions=comps)["rows"]}
    total = rows["Total balance with other banks"]
    parts = {c["field"] for c in total["breakdown"]["components"]}
    assert parts == {"In Bangladesh", "Outside Bangladesh"} and total["breakdown"]["report"] == "2021-22"
    assert rows["In Bangladesh"]["breakdown_of"] == [total["series_id"]]


def test_a_deducted_part_is_subtracted_once_even_when_printed_negative():
    md = "| Particulars | 2022 |\n|---|---|\n| Receipts | 100 |\n| Payments | (30) |\n| Net | 70 |\n"
    payload = {"t": [{"n": 1, "s": "C", "st": "x", "sc": "S", "sh": "x", "lc": 1, "nc": 0, "y": [{"c": 2, "y": 2022}],
                      "k": [{"d": "R", "t": 3, "p": [1, 2], "n": [2]}], "r": [{"r": i, "c": "=", "l": "L"} for i in (1, 2, 3)]}]}
    (c,) = ext.decode_compositions(payload, md)
    assert c["status"] == "MATCH" and c["parts_sum"] == 70.0


def test_ratio_percent_and_product_breakdowns_are_checked_by_their_operation():
    md = ("| Particulars | Note | 2022 |\n|---|---|---|\n| Net profit | | 7,609,276,049 |\n| Shares | | 1,460,000,000 |\n"
          "| EPS (i/ii) | | 5.21 |\n| Wrong EPS | | 9.99 |\n| Margin % | | 12.5 |\n| Revenue | | 800 |\n| Profit | | 100 |\n"
          "| Qty | | 4 |\n| Price | | 25 |\n| Amount | | 100 |\n")
    rows = [{"r": i, "c": "=", "l": "L"} for i in range(1, 11)]
    payload = {"t": [{"n": 1, "s": "N", "st": "x", "sc": "S", "sh": "x", "lc": 1, "nc": 2, "y": [{"c": 3, "y": 2022}], "r": rows,
                      "k": [{"d": "R", "t": 3, "p": [1, 2], "o": "/"}, {"d": "R", "t": 4, "p": [1, 2], "o": "/"},
                            {"d": "R", "t": 5, "p": [7, 6], "o": "%"}, {"d": "R", "t": 10, "p": [8, 9], "o": "*"},
                            {"d": "R", "t": 3, "p": [1], "o": "/"}]}]}
    comps = {c["total"]["row"]: c for c in ext.decode_compositions(payload, md)}
    assert comps[3]["operation"] == "RATIO" and comps[3]["status"] == "MATCH"        # 7.609bn / 1.46bn = 5.2118
    assert comps[4]["status"] == "MISMATCH"
    assert comps[5]["operation"] == "PERCENT" and comps[5]["status"] == "MATCH"      # 100 / 800 * 100 = 12.5
    assert comps[10]["operation"] == "PRODUCT" and comps[10]["status"] == "MATCH"    # 4 * 25 = 100
    assert len(comps) == 4                                                            # a ratio with one part is dropped


def _cell_item(field, value, entity=None, row=1):
    return {"concept": {"name": field, "statement": "INCOME_STATEMENT", "category": "Profit"},
            "scope": {"type": "CONSOLIDATED", "entity": entity}, "raw_field": field,
            "data": [{"year": 2021, "numeric_value": value, "unit": "BDT", "multiplier": 1.0, "table_num": 1,
                      "table_row": row, "table_col": 3}]}


def test_cell_details_of_a_series_gives_that_lines_cell_only(ocr):  # noqa: F811
    database.upsert_page(None, "CELLCO", "2020-21", 3, "", is_selected=True)
    database.update_page_ocr("CELLCO", "2020-21", 3, MD)
    database.save_extracted_metrics("CELLCO", "2020-21", 3, [
        _cell_item("Net profit after tax", 100, row=1),
        _cell_item("Net profit after tax (Conventional)", 70, row=2),
        _cell_item("Balance with Bangladesh Bank and its agent bank(s)", 55, row=3),
        _cell_item("Non-controlling interest", 9, row=4),
        _cell_item("Revenue", 40, entity="City Brokerage Limited", row=5),
    ])

    assert ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "CELLCO"}).status_code == 200
    merged = {r["base_field"]: r["series_id"] for r in full_results(ocr, "CELLCO")["merged_rows"]}

    def values(field):
        body = ocr.get("/ocr/company/CELLCO/cell-details", headers=KEY,
                       params={"series_id": merged[field], "year": 2021}).json()
        return [r["effective_value"] for r in body["cell_records"]]

    assert values("Net profit after tax (Conventional)") == [70.0]       # not the plain line
    assert values("Net profit after tax") == [100.0]
    assert values("Balance with Bangladesh Bank and its agent bank(s)") == [55.0]
    assert values("Non-controlling interest") == [9.0]
    assert [v for f, v in ((f, values(f)) for f in merged if f.startswith("Revenue")) ][0] == [40.0]   # a subsidiary's own row


def test_results_and_cell_details_give_the_same_dots_from_the_rows_own_scope(ocr):  # noqa: F811
    def item(scope, values):
        return {"concept": {"name": "Weighted average number of shares", "statement": "NOTES_AND_FOOTNOTES",
                            "category": "EPS"}, "scope": {"type": scope}, "raw_field": "Weighted average number of shares",
                "data": [{"year": y, "numeric_value": v, "unit": "Nos.", "multiplier": 1.0, "table_num": 1,
                          "table_row": 1 if scope == "STANDALONE" else 2, "table_col": 3 + i}
                         for i, (y, v) in enumerate(values)]}
    for i, fy in enumerate(("2014-15", "2015-16", "2016-17")):
        database.upsert_page(None, "DOTCO", fy, 10 + i, "", is_selected=True)
        database.update_page_ocr("DOTCO", fy, 10 + i, MD)
    # STANDALONE: 2014 printed by two reports (restated: yellow); CONSOLIDATED: three reports
    database.save_extracted_metrics("DOTCO", "2014-15", 10, [item("STANDALONE", [(2015, 5), (2014, 4)]),
                                                             item("CONSOLIDATED", [(2015, 9), (2014, 8)])])
    database.save_extracted_metrics("DOTCO", "2015-16", 11, [item("STANDALONE", [(2016, 6), (2015, 5)]),
                                                             item("CONSOLIDATED", [(2016, 7), (2015, 1)])])
    database.save_extracted_metrics("DOTCO", "2016-17", 12, [item("STANDALONE", [(2017, 7), (2016, 6)]),
                                                             item("CONSOLIDATED", [(2017, 3), (2016, 2)])])
    assert ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "DOTCO"}).status_code == 200
    res = full_results(ocr, "DOTCO")
    for row in res["raw_rows"]:
        assert row["restatement_dots"] == {}
    for row in res["merged_rows"]:
        for y in ("2015", "2016"):
            cd = ocr.get("/ocr/company/DOTCO/cell-details", headers=KEY, params={
                "series_id": row.get("series_id"), "year": int(y)}).json()
            color = {"RESTATED_IDENTICAL": "green", "RESTATED_LOW": "yellow", "RESTATED_MID": "yellow", "RESTATED_HIGH": "yellow"}
            assert row["restatement_dots"].get(y, []) == [color[e["type"]] for e in cd["flags"]], (row["scope"], y)
            assert row["flags"].get(y, []) == cd["flags"], (row["scope"], y)       # the table and the cell give the same entries
    std = next(r for r in res["merged_rows"] if r["scope"] == "STANDALONE")
    con = next(r for r in res["merged_rows"] if r["scope"] == "CONSOLIDATED")
    assert std["restatement_dots"] == {"2015": ["green"], "2016": ["green"]}   # never counted across scopes
    assert con["restatement_dots"] == {"2015": ["yellow"], "2016": ["yellow"]}


def test_note_cells_are_marked_from_the_note_column_and_label_references(ocr):  # noqa: F811
    md = ("| Particulars | Note | 2022 |\n|---|---|---|\n| Cash | 4 | 100 |\n| The Bank (note 42.a) | | 50 |\n"
          "| Receivables | 5, 6 | 70 |\n| Other | | 9 |\n")
    payload = {"t": [{"n": 1, "s": "B", "lc": 1, "nc": 2, "y": [{"c": 3, "y": 2022}],
                      "r": [{"r": i, "c": "=", "l": "L"} for i in (1, 2, 3, 4)]}]}
    items = {i["raw_field"]: i for i in ext.decode_compact_extraction(payload, md, report_end=2022)}
    assert items["Cash"]["note_cell"] == {"table_num": 1, "table_row": 1, "table_col": 2} and items["Cash"]["notes"] == ["4"]
    assert items["The Bank (note 42.a)"]["notes"] == ["42.a"]
    assert items["The Bank (note 42.a)"]["note_cell"]["table_col"] == 1          # the label cell
    assert items["Other"]["note_cell"] is None
    database.upsert_page(None, "NOTECO", "2021-22", 4, "", is_selected=True)
    database.update_page_ocr("NOTECO", "2021-22", 4, md)
    database.save_extracted_metrics("NOTECO", "2021-22", 4, list(items.values()))
    # a heading row with a note but no figures is not a metric, yet its note cell is recorded
    md2 = md + "| **Provision for taxation** | 44 | |\n"
    database.update_page_ocr("NOTECO", "2021-22", 4, md2)
    database.save_extracted_metrics("NOTECO", "2021-22", 4, list(items.values()))
    database.save_page_note_cells("NOTECO", "2021-22", 4, ext.decode_note_cells(payload, md2))
    cells = ocr.get("/ocr/company/NOTECO/note-cells", headers=KEY).json()["2021-22_4"]
    assert [(c["table_row"], c["table_col"], c["notes"]) for c in cells] == [
        (1, 2, ["4"]), (2, 1, ["42.a"]), (3, 2, ["5", "6"]), (5, 2, ["44"])]
    assert cells[3]["metric_ids"] == [] and cells[3]["raw_field"] == "Provision for taxation"
    assert cells[0]["metric_ids"] and cells[0]["raw_field"] == "Cash"


SEG_MD = """| Particulars | Note | Conventional Taka 2013 | Islamic Taka 2013 | Offshore Taka 2013 | Total Taka 2013 |
|---|---|---|---|---|---|
| Segment liabilities | | 135,026 | 4,464 | 7,980 | 147,470 |
"""


def test_same_year_columns_are_separate_cells_named_after_their_header(ocr):  # noqa: F811
    payload = {"t": [{"n": 1, "s": "N", "lc": 1, "nc": 2,
                      "y": [{"c": 3, "y": 2013}, {"c": 4, "y": 2013}, {"c": 5, "y": 2013}, {"c": 6, "y": 2013}],
                      "r": [{"r": 1, "c": "=", "l": "G"}]}]}
    items = ext.decode_compact_extraction(payload, SEG_MD, report_end=2013)
    assert sorted((i["scope"]["segment"] or "-", i["data"][0]["numeric_value"]) for i in items) == [
        ("-", 147470.0), ("Conventional", 135026.0), ("Islamic", 4464.0), ("Offshore", 7980.0)]
    database.upsert_page(None, "SEGCO", "2012-13", 9, "", is_selected=True)
    database.update_page_ocr("SEGCO", "2012-13", 9, SEG_MD)
    database.save_extracted_metrics("SEGCO", "2012-13", 9, items)
    res = full_results(ocr, "SEGCO")
    names = sorted(r["field"] for r in res["raw_rows"])
    assert names == ["Segment liabilities", "Segment liabilities (Conventional)", "Segment liabilities (Islamic)",
                     "Segment liabilities (Offshore)"]
    assert ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "SEGCO"}).status_code == 200
    merged = {r["field"]: r["series_id"] for r in full_results(ocr, "SEGCO")["merged_rows"]}
    for field, value in (("Segment liabilities", 147470.0), ("Segment liabilities (Islamic)", 4464.0)):
        cd = ocr.get("/ocr/company/SEGCO/cell-details", headers=KEY, params={
            "series_id": merged[field], "year": 2013}).json()
        assert [r["effective_value"] for r in cd["cell_records"]] == [value]          # the cell itself, nothing else
        assert {r["status"] for r in cd["cell_records"]} == {"ORIGINAL_STATED"}
    assert sl.display_name("Segment liabilities", "STANDALONE", None, "Islamic") == "Segment liabilities - Islamic"


def test_a_row_repeated_per_segment_and_a_bogus_scale_do_not_corrupt_the_cells():
    payload = {"t": [{"n": 1, "s": "N", "lc": 1, "nc": 2, "m": 1e-15,
                      "y": [{"c": 3, "y": 2013}, {"c": 4, "y": 2013}, {"c": 5, "y": 2013}, {"c": 6, "y": 2013}],
                      "r": [{"r": 1, "c": "Segment liabilities (Conventional)", "l": "G", "g": "Conventional"},
                            {"r": 1, "c": "Segment liabilities (Islamic)", "l": "G", "g": "Islamic"},
                            {"r": 1, "c": "Segment liabilities (Offshore)", "l": "G", "g": "Offshore"}]}]}
    items = ext.decode_compact_extraction(payload, SEG_MD, report_end=2013)
    got = sorted((i["scope"]["segment"] or "-", i["concept"]["name"], i["data"][0]["numeric_value"], i["data"][0]["multiplier"])
                 for i in items)
    assert got == [("-", "Segment liabilities", 147470.0, 1.0), ("Conventional", "Segment liabilities", 135026.0, 1.0),
                   ("Islamic", "Segment liabilities", 4464.0, 1.0), ("Offshore", "Segment liabilities", 7980.0, 1.0)]


def test_second_printing_in_the_same_report_is_not_a_restatement():
    from app.ocr import router as r
    recs = [{"report_year": "2012-13", "page_number": 49, "table_num": 1, "effective_value": 5.0},
            {"report_year": "2012-13", "page_number": 50, "table_num": 1, "effective_value": 5.0},
            {"report_year": "2012-13", "page_number": 52, "table_num": 2, "effective_value": 4.0},
            {"report_year": "2013-14", "page_number": 126, "table_num": 1, "effective_value": 6.0}]
    reps = r._publications(recs)
    assert r._publication_statuses(reps) == ["ORIGINAL_STATED", "REPEATED_IN_REPORT", "DUPLICATE_IN_REPORT", "RESTATED"]
    assert r._restatement_dots(recs) == ["yellow"]          # only the later report gets a dot
    assert r._duplicate_in_report(recs)                      # a different figure in the same report: review
    assert not r._duplicate_in_report(recs[:2])              # the same figure printed again: fine


def test_same_name_under_two_headings_in_one_report_is_never_compared():
    from app.ocr import router as r
    def rec(cat, val, rep="2023-24"):
        return {"report_year": rep, "page_number": 444, "table_num": 1 if "assets" in cat else 2, "category": cat,
                "effective_value": val}
    rows = [rec("(Increase) / decrease of other assets", 1.0), rec("Increase/ (decrease) of other liabilities", 2.0),
            rec("Increase/(decrease) of other assets", 1.5, "2024-25")]
    kept = r._narrow_by_category(rows, "Increase/ (decrease) of other liabilities")
    assert [k["effective_value"] for k in kept] == [2.0, 1.5]          # the other report has one heading: untouched
    elsewhere = dict(rec("Cash", 3.0), page_number=52)                 # the same name on ANOTHER page stays comparable
    assert elsewhere in r._narrow_by_category(rows[:2] + [elsewhere], "Increase/ (decrease) of other liabilities")
    assert not r._duplicate_in_report(kept[:1])
    assert r._narrow_by_category(rows, None) == rows


def test_merge_reviews_a_line_printed_twice_in_one_report_only_when_the_figures_differ():
    def metric(page, row):
        return {"id": page, "fiscal_year": "2012-13", "page_number": page, "statement_type": "BALANCE_SHEET",
                "scope": "CONSOLIDATED", "field": "Total assets", "raw_field": "Total assets", "category": "Assets",
                "aggregation_level": "GRAND_TOTAL", "metric_type": "CURRENCY",
                "data": [{"year": 2013, "numeric_value": 100.0, "unit": "BDT", "multiplier": 1.0,
                          "table_num": 1, "table_row": row, "table_col": 3}]}
    (node,) = sl.build_nodes([metric(49, 20), metric(52, 18)], {})
    assert node["flags"] == []                               # same figure printed twice: fine
    other = metric(52, 18)
    other["data"][0]["numeric_value"] = 90.0
    (node,) = sl.build_nodes([metric(49, 20), other], {})
    assert node["flags"]                                     # a different figure: review


def test_collision_details_say_which_printed_cell_each_value_came_from():
    prov_a = {"page_number": 81, "table_num": 2, "table_row": 4, "table_col": 4}
    prov_b = {"page_number": 193, "table_num": 1, "table_row": 6, "table_col": 3}
    h = {"report": "2012-13", "prov": {2013: prov_a}}
    n = {"report": "2013-14", "prov": {2013: prov_b}}
    src = sl._overlap_sources(h, n, [{"year": 2013}])
    assert src == {"2013": {"a_report": "2012-13", "a_page_number": 81, "a_table_num": 2, "a_table_row": 4, "a_table_col": 4,
                            "b_report": "2013-14", "b_page_number": 193, "b_table_num": 1, "b_table_row": 6, "b_table_col": 3}}
    item = {"kind": "VALUE_CONFLICT", "payload": {
        "a_field": "A", "b_field": "B", "statement_type": "NOTES_AND_FOOTNOTES", "a_report": "2012-13", "b_report": "2013-14",
        "evidence": {"overlap": [{"year": 2013, "a": 1.0, "b": 5.0, "match": False}]}, "overlap_sources": src}}
    (conflict,) = sl.value_conflicts_from_review([item])
    (detail,) = conflict["collision_details"]
    assert detail["a_page_number"] == 81 and detail["b_table_col"] == 3 and detail["val_a"] == 1.0


# ---------------------------------------------------------------- node identity (same label in two places)
def _opex(fy, cat, values, row, raw=None, field="Rent"):
    m = _m(fy, field, values, category=cat, stmt="INCOME_STATEMENT", page=row, row=row)
    if raw:
        m["raw_field"] = raw
    return m


def test_same_label_under_two_headings_is_two_nodes_and_links_by_heading():
    metrics = [
        _opex("2021-22", "Administrative Expenses", {2022: 100.0}, 1), _opex("2021-22", "Selling Expenses", {2022: 200.0}, 2),
        _opex("2022-23", "Administrative Expenses", {2023: 110.0, 2022: 100.0}, 3),
        _opex("2022-23", "Selling Expenses", {2023: 210.0, 2022: 200.0}, 4),
    ]
    nodes, res, series, links, review = _run(metrics)
    assert len(nodes) == 4 and not any(n["flags"] for n in nodes)
    assert len({n["disc"] for n in nodes}) == 2 and all(n["disc"].startswith("c:") for n in nodes)
    assert sorted(len(s["nodes"]) for s in series) == [2, 2]
    assert all(len({n["cat"] for n in s["nodes"]}) == 1 for s in series)


def test_rows_of_a_record_list_with_one_concept_name_are_separate_nodes():
    cat = "Particulars of Disposal"
    metrics = [_opex("2021-22", cat, {2022: 100.0}, 1, raw="Yamaha"), _opex("2021-22", cat, {2022: 900.0}, 2, raw="Toyota"),
               _opex("2022-23", cat, {2023: 50.0}, 3, raw="Honda"), _opex("2022-23", cat, {2023: 70.0}, 4, raw="Nissan")]
    nodes, res, series, links, review = _run(metrics)
    assert len(nodes) == 4 and not any(n["flags"] for n in nodes)
    assert all(n["disc"].startswith("r:") for n in nodes)
    assert not links                     # different rows are never paired by the shared concept name


def test_identical_rows_still_raise_a_conflict():
    cat = "Particulars of Disposal"
    metrics = [_opex("2021-22", cat, {2022: 100.0}, 1, raw="Toyota"), _opex("2021-22", cat, {2022: 900.0}, 2, raw="Toyota")]
    nodes = sl.build_nodes(metrics, {})
    assert len(nodes) == 1 and nodes[0]["flags"] == ["INTRA_NODE_CONFLICT"]


def test_unique_label_keeps_the_plain_node_id():
    nodes = sl.build_nodes([_opex("2021-22", "Administrative Expenses", {2022: 100.0}, 1)], {})
    assert nodes[0]["id"] == "2021-22|rent|OPEX" and nodes[0]["disc"] == ""


def test_unique_head_links_to_the_node_under_the_same_heading():
    metrics = [
        _opex("2021-22", "Selling Expenses", {2022: 200.0}, 1),
        _opex("2022-23", "Administrative Expenses", {2023: 110.0}, 2), _opex("2022-23", "Selling Expenses", {2023: 210.0, 2022: 200.0}, 3),
    ]
    nodes, res, series, links, review = _run(metrics)
    assert len(links) == 1 and links[0]["to_node"].endswith("|c:selling expense")


def test_lines_told_apart_by_heading_are_never_offered_to_gemini_or_joined():
    cat_a, cat_b = "Opening balances", "Closing balances"
    metrics = [
        _opex("2021-22", cat_a, {2022: 100.0}, 1, field="Equity"), _opex("2021-22", cat_b, {2022: 300.0}, 2, field="Equity"),
        _opex("2023-24", cat_a, {2024: 150.0}, 3, field="Equity Balance"), _opex("2023-24", cat_b, {2024: 350.0}, 4, field="Equity Balance"),
    ]
    nodes = sl.build_nodes(metrics, {})
    a = [n for n in nodes if n["report"] == "2021-22"]
    b = [n for n in nodes if n["report"] == "2023-24"]
    assert all(n["disc"] for n in a) and all(n["disc"] for n in b)
    assert sl.veto(a[0], a[1], lambda *_: False, sl._Overrides([], a[0]["partition"]))   # not the same line
    judge = RecordingJudge(choose=0)
    sl.link_company(nodes, [], judge=judge)
    for q in judge.questions:
        assert all(not sl._disc_clash(q["head_node"], c["node"]) for c in q["candidates"])


def test_gemini_sees_heading_and_row_label_for_told_apart_lines():
    cat = "Particulars of Disposal"
    nodes = sl.build_nodes([_opex("2021-22", cat, {2022: 100.0}, 1, raw="Yamaha"),
                            _opex("2021-22", cat, {2022: 900.0}, 2, raw="Toyota")], {})
    shown = {sl._shown(n) for n in nodes}
    assert any("row: Yamaha" in x and "heading: Particulars of Disposal" in x for x in shown)
    plain = sl.build_nodes([_opex("2021-22", cat, {2022: 100.0}, 1)], {})
    assert sl._shown(plain[0]) == "Rent"


# ---------------------------------------------------------------- gap links (R4) safety checks
def _gap_metrics(cat_a, cat_b, v_a=100.0, v_b=100.0, field="Interest Income", raw_a=None, raw_b=None, extra=None):
    ms = [_opex("2018-19", cat_a, {2019: v_a}, 1, field=field, raw=raw_a),
          _opex("2020-21", "Other Income", {2021: 7.0}, 2, field="Filler Line"),     # the report in between prints other lines
          _opex("2021-22", cat_b, {2022: v_b}, 3, field=field, raw=raw_b)]
    return ms + (extra or [])


def _gap_run(metrics, overrides=()):
    nodes = sl.build_nodes(metrics, {})
    res = sl.link_company(nodes, list(overrides), judge=None, max_gap=3)
    return nodes, res


def test_gap_link_across_unrelated_headings_becomes_a_review_item():
    nodes, res = _gap_run(_gap_metrics("Related party transactions", "Other income statement"))
    assert not [l for l in res["links"] if l["rule"] in ("GAP_NAME", "LONG_GAP")]
    items = [i for i in res["review_items"] if i["kind"] == "GAP_BLOCKED"]
    assert len(items) == 1 and "headings share no word" in items[0]["payload"]["reason"]


def test_gap_link_with_a_shared_heading_word_is_made():
    nodes, res = _gap_run(_gap_metrics("45 Consolidated Other income", "Other income details"))
    assert [l["rule"] for l in res["links"] if l["rule"] in ("GAP_NAME", "LONG_GAP")] == ["GAP_NAME"]
    assert not [i for i in res["review_items"] if i["kind"] == "GAP_BLOCKED"]


def test_gap_link_blocked_by_a_hard_veto_goes_to_review():
    nodes, res = _gap_run(_gap_metrics("Other income", "Other income", field="Deposits"), overrides=[])
    # same label, so no veto: linked. A hard contrast between different names is blocked instead
    a = _m("2018-19", "Current Assets Other", {2019: 5.0}, category="Other income", stmt="INCOME_STATEMENT", page=1)
    b = _m("2021-22", "Non-current Assets Other", {2022: 5.0}, category="Other income", stmt="INCOME_STATEMENT", page=3)
    a["raw_field"] = b["raw_field"] = "Assets Other Items"
    filler = _m("2020-21", "Filler Line", {2021: 7.0}, category="Other income", stmt="INCOME_STATEMENT", page=2)
    nodes, res = _gap_run([a, filler, b])
    assert not [l for l in res["links"] if l["rule"] in ("GAP_NAME", "LONG_GAP")]
    assert [i["payload"]["reason"] for i in res["review_items"] if i["kind"] == "GAP_BLOCKED"]


def test_entity_wording_in_one_name_is_not_a_veto_when_the_printed_row_label_is_shared():
    h = sl.build_nodes([_opex("2018-19", "Increase of other assets", {2019: 5.0}, 1,
                              field="City Brokerage Limited increase of other assets", raw="City Brokerage Limited")], {})[0]
    n = sl.build_nodes([_opex("2021-22", "Increase of other assets", {2022: 5.0}, 1,
                              field="Other assets increase", raw="City Brokerage Limited")], {})[0]
    assert sl.veto(h, n, lambda *_: False, sl._Overrides([], h["partition"])) is None


def test_gap_link_with_a_huge_jump_is_kept_and_flagged_but_a_sign_flip_is_not():
    nodes, res = _gap_run(_gap_metrics("Other income", "Other income", v_a=100.0, v_b=100_000.0))
    gap = [l for l in res["links"] if l["rule"] in ("GAP_NAME", "LONG_GAP")]
    assert len(gap) == 1 and gap[0]["evidence"]["magnitude_ratio"] == 1000.0
    assert [i["kind"] for i in res["review_items"] if i["kind"] == "MAGNITUDE_JUMP"] == ["MAGNITUDE_JUMP"]
    nodes, res = _gap_run(_gap_metrics("Other income", "Other income", v_a=100.0, v_b=-300.0))
    assert not [i for i in res["review_items"] if i["kind"] == "MAGNITUDE_JUMP"]


def test_user_force_link_applies_across_a_gap_and_force_split_prevents_it():
    ms = _gap_metrics("Related party transactions", "Other income statement")
    nodes, res = _gap_run(ms)
    assert not [l for l in res["links"] if l["rule"] == "USER"]
    base = {"id": 1, "kind": "FORCE_LINK", "statement_type": "INCOME_STATEMENT", "scope": "STANDALONE",
            "entity_key": "", "segment_key": "", "a_label_key": "interest income", "a_report": "2018-19",
            "b_label_key": "interest income", "b_report": "2021-22", "created_at": ""}
    nodes, res = _gap_run(ms, overrides=[base])
    assert [l["rule"] for l in res["links"] if l["to_report"] == "2021-22"] == ["USER"]
    nodes, res = _gap_run(ms, overrides=[{**base, "kind": "FORCE_SPLIT"}])
    assert not [l for l in res["links"] if l["to_report"] == "2021-22"]


# ---------------------------------------------------------------- timelines on SPLIT / MERGE items
def _four_reports(single_first):
    """One line in the first two reports, two lines in the last two (or the mirror when single_first is False)."""
    one = [_m("2020-21", "Others Liabilities", {2021: 3_000_000, 2020: 2_900_000}),
           _m("2021-22", "Others Liabilities", {2022: 3_300_000, 2021: 3_000_000})]
    two = []
    for fy, ys in (("2022-23", (2023, 2022)), ("2023-24", (2024, 2023))):
        for field, base in (("Other payables", 2_000_000), ("Unclaimed dividend", 1_000_000)):
            two.append(_m(fy, field, {ys[0]: base + ys[0] - 2000, ys[1]: base + ys[1] - 2000}))
    # make the overlap year (2022) add up: 2_000_022 + 1_000_022 is not 3_300_000, so set explicitly
    two = [_m("2022-23", "Other payables", {2023: 2_300_000, 2022: 2_200_000}),
           _m("2022-23", "Unclaimed dividend", {2023: 1_100_000, 2022: 1_100_000}),
           _m("2023-24", "Other payables", {2024: 2_400_000, 2023: 2_300_000}),
           _m("2023-24", "Unclaimed dividend", {2024: 1_200_000, 2023: 1_100_000})]
    if single_first:
        return one + two
    # mirror: two lines early, one line late
    early = [_m("2020-21", "Other payables", {2021: 2_000_000, 2020: 1_900_000}),
             _m("2020-21", "Unclaimed dividend", {2021: 1_000_000, 2020: 1_000_000}),
             _m("2021-22", "Other payables", {2022: 2_200_000, 2021: 2_000_000}),
             _m("2021-22", "Unclaimed dividend", {2022: 1_100_000, 2021: 1_000_000})]
    late = [_m("2022-23", "Others Liabilities", {2023: 3_400_000, 2022: 3_300_000}),
            _m("2023-24", "Others Liabilities", {2024: 3_600_000, 2023: 3_400_000})]
    return early + late


def _timeline_item(metrics, kind):
    nodes, res, series, links, review = _run(metrics)
    series_merge._attach_timelines(res["review_items"], series, nodes)
    (item,) = [i for i in res["review_items"] if i["kind"] == kind]
    return item


def test_split_item_carries_the_whole_history_of_every_line():
    item = _timeline_item(_four_reports(True), "SPLIT_DETECTED")
    p = item["payload"]
    assert p["from"]["field"] == "Others Liabilities" and len(p["into"]) == 2 and p["year"] == 2022   # unchanged keys
    tl = p["timeline"]
    assert tl["years"] == [2020, 2021, 2022, 2023, 2024]
    assert tl["reports"] == ["2020-21", "2021-22", "2022-23", "2023-24"]
    (frm,) = [l for l in tl["lines"] if l["role"] == "from"]
    ins = [l for l in tl["lines"] if l["role"] == "into"]
    assert len(ins) == 2
    assert frm["values"] == {"2020": 2_900_000.0, "2021": 3_000_000.0, "2022": 3_300_000.0, "2023": None, "2024": None}
    assert frm["first_report"] == "2020-21" and frm["last_report"] == "2021-22"
    assert frm["stating_reports"] == {"2020": "2020-21", "2021": "2021-22", "2022": "2021-22"}
    assert [m["report"] for m in frm["members"]] == ["2020-21", "2021-22"]
    assert frm["members"][0]["values"] == {"2020": 2_900_000.0, "2021": 3_000_000.0} and frm["members"][0]["page_number"] == 1
    op = next(l for l in ins if l["field"] == "Other payables")
    assert op["values"] == {"2020": None, "2021": None, "2022": 2_200_000.0, "2023": 2_300_000.0, "2024": 2_400_000.0}
    assert op["stating_reports"]["2023"] == "2023-24" and op["first_report"] == "2022-23"
    assert [m["report"] for m in op["members"]] == ["2022-23", "2023-24"]
    assert all(isinstance(l["node_id"], str) for l in tl["lines"])
    import json
    json.dumps(p)


def test_merge_item_carries_the_whole_history_of_every_line():
    item = _timeline_item(_four_reports(False), "MERGE_DETECTED")
    p = item["payload"]
    assert p["into"]["field"] == "Others Liabilities" and len(p["from"]) == 2
    tl = p["timeline"]
    assert tl["years"] == [2020, 2021, 2022, 2023, 2024]
    frm = [l for l in tl["lines"] if l["role"] == "from"]
    (into,) = [l for l in tl["lines"] if l["role"] == "into"]
    assert [l["role"] for l in tl["lines"]] == ["from", "from", "into"]
    assert {l["field"] for l in frm} == {"Other payables", "Unclaimed dividend"}
    assert into["values"] == {"2020": None, "2021": None, "2022": 3_300_000.0, "2023": 3_400_000.0, "2024": 3_600_000.0}
    op = next(l for l in frm if l["field"] == "Other payables")
    assert op["values"]["2020"] == 1_900_000.0 and op["values"]["2023"] is None
    assert [m["report"] for m in into["members"]] == ["2022-23", "2023-24"]


def test_review_endpoint_serves_the_timeline_and_refreshes_it(ocr, monkeypatch):  # noqa: F811
    _seed("TLCO", "2020-21", 5, [_item("Others Liabilities", [(2021, 3_000_000), (2020, 2_900_000)])])
    _seed("TLCO", "2021-22", 6, [_item("Others Liabilities", [(2022, 3_300_000), (2021, 3_000_000)])])
    _seed("TLCO", "2022-23", 7, [_item("Other payables", [(2023, 2_300_000), (2022, 2_200_000)]),
                                 _item("Unclaimed dividend", [(2023, 1_100_000), (2022, 1_100_000)])])
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: RecordingJudge())
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "TLCO"})
    items = ocr.get("/ocr/company/TLCO/merge/review", headers=KEY).json()["items"]
    (item,) = [i for i in items if i["kind"] == "SPLIT_DETECTED"]
    tl = item["payload"]["timeline"]
    assert tl["years"] == [2020, 2021, 2022, 2023] and len(tl["lines"]) == 3
    sids = {s["id"] for s in ocr.get("/ocr/company/TLCO/series", headers=KEY).json()["series"]}
    assert all(l["series_id"] in sids for l in tl["lines"])

    # a later report extends the history; the merge items are replaced by the new merge's (new ids)
    _seed("TLCO", "2023-24", 8, [_item("Other payables", [(2024, 2_400_000), (2023, 2_300_000)]),
                                 _item("Unclaimed dividend", [(2024, 1_200_000), (2023, 1_100_000)])])
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "TLCO"})
    items = ocr.get("/ocr/company/TLCO/merge/review", headers=KEY).json()["items"]
    (again,) = [i for i in items if i["kind"] == "SPLIT_DETECTED"]
    assert again["payload"]["timeline"]["years"] == [2020, 2021, 2022, 2023, 2024]


# ---------------------------------------------------------------- split / merge detection (M6)
def test_split_uses_the_lines_own_overlap_year():
    # The partition-wide latest common year is 2020, but the split line only prints 2019:
    # a partition-wide year would never test it.
    heads = sl.build_nodes([_m("2019-20", "Trade and Other Payables", {2019: 3_000_000}),
                            _m("2019-20", "Accrued Expenses", {2020: 9_100_000, 2019: 8_000_000}, row=2)], {})
    new = sl.build_nodes([_m("2020-21", "Trade Payables", {2020: 1_500_000, 2019: 2_000_000}, row=1),
                          _m("2020-21", "Other Payables", {2020: 700_000, 2019: 1_000_000}, row=2)], {})
    items = sl._detect_splits(heads, new)
    assert len(items) == 1 and items[0]["payload"]["year"] == 2019
    assert items[0]["payload"]["from"]["label_key"] == "trade other payable"
    assert sorted(i["label_key"] for i in items[0]["payload"]["into"]) == ["other payable", "trade payable"]


def test_split_parts_are_found_beyond_the_first_twelve_candidates():
    head = sl.build_nodes([_m("2019-20", "Investment in Shares", {2020: 36_507_658}, category="Investments")], {})
    fillers = [_m("2020-21", f"Filler Line {i:02d}", {2020: 1_000_000_000 + i * 7_777_777}, category="Investments", row=i)
               for i in range(1, 15)]
    parts = [_m("2020-21", "Investment in Shares of Associates", {2020: 30_749_850}, category="Investments", row=20),
             _m("2020-21", "Investment in Shares Other", {2020: 5_757_808}, category="Investments", row=21)]
    new = sl.build_nodes(fillers + parts, {})
    items = sl._detect_splits(head, new)
    assert len(items) == 1
    assert sorted(i["value"] for i in items[0]["payload"]["into"]) == [5_757_808, 30_749_850]


# ---------------------------------------------------------------- label key variants (M5)
@pytest.mark.parametrize("a,b", [
    ("Balance with Bangladesh Bank and its agent bank(s) (including foreign currencies)",
     "Balance with Bangladesh Bank and its agent banks (including foreign currencies)"),
    ("Balance with Bangladesh Bank and its agent bank (including foreign currencies)",
     "Balance with Bangladesh Bank and its agent banks (including foreign currencies)"),
    ("31. Other Comprehensive Income", "Other Comprehensive Income"),
    ("Note 12 Loans", "Loans"),
    ("Profit/(Loss) Attributable to Owners of the Company", "Profit Attributable to Owners of the Company"),
    ("Dividend equalization reserve", "Dividend equalisation reserve"),
    ("EPS EPS", "EPS"),
    ("Long term borrowing", "Long term borrowings"),
    ("Income tax expense", "Income tax expenses"),
    ("Total liabilities", "Total Liabilities"),
])
def test_label_key_folds_real_variants(a, b):
    assert sk.label_key(a) == sk.label_key(b)


@pytest.mark.parametrize("a,b", [
    ("Increase/(decrease) of other liabilities", "Other liabilities"),     # a movement is not the balance
    ("Total assets", "Total liabilities"),
    ("Current assets", "Non-current assets"),
    ("Share capital", "Share premium"),
    ("2023 dividend", "2022 dividend"),                                     # years are not note numbers
    ("18 months deposits", "6 months deposits"),
])
def test_label_key_does_not_merge_different_lines(a, b):
    assert sk.label_key(a) != sk.label_key(b)


def test_total_is_kept_in_the_key_but_ignored_when_matching_across_reports():
    assert sk.label_key("Cash total") != sk.label_key("Cash")
    assert sk.core_key(sk.label_key("Total cash")) == sk.core_key(sk.label_key("Cash total")) == "cash"
    metrics = [_m("2020-21", "Deposits and Other Accounts Total", {2021: 5_000_000_000}, category="Liabilities"),
               _m("2021-22", "Deposits and Other Accounts", {2022: 6_000_000_000, 2021: 5_000_000_000}, category="Liabilities")]
    nodes, res, series, links, review = _run(metrics)
    assert [l["rule"] for l in links] == ["NAME_EXACT"] and len(series) == 1


def test_a_part_and_its_subtotal_printed_together_are_never_matched_by_total():
    metrics = [_m("2020-21", "Cash", {2021: 100.0}, category="Assets", row=1),
               _m("2020-21", "Cash Total", {2021: 250.0}, category="Assets", row=2),
               _m("2021-22", "Cash", {2022: 120.0, 2021: 100.0}, category="Assets", row=1),
               _m("2021-22", "Cash Total", {2022: 300.0, 2021: 250.0}, category="Assets", row=2)]
    nodes, res, series, links, review = _run(metrics)
    assert sorted(len(s["nodes"]) for s in series) == [2, 2]       # each line continues itself, never the other


# ---------------------------------------------------------------- Gemini verdicts (M13)
class VerdictJudge:
    def __init__(self, verdict, choice=0):
        self.verdict, self.choice, self.questions = verdict, choice, []

    def __call__(self, questions):
        self.questions.extend(questions)
        conf = {"SAME": 0.95, "UNSURE": 0.5}.get(self.verdict, 0.0)
        return {q["qid"]: {"choice": -1 if self.verdict in ("DIFFERENT", "NO_ANSWER") else self.choice, "verdict": self.verdict,
                           "confidence": conf, "type_head": "", "type_choice": ""} for q in questions}


_VERDICT_METRICS = [_m("2019-20", "Advance Income Tax", {2020: 160_281_671}, category="Tax"),
                    _m("2020-21", "Tax Deducted at Source", {2021: 170_000_000, 2020: 160_281_671}, category="Other")]


@pytest.mark.parametrize("verdict,rule,item", [("SAME", "GEMINI", None), ("UNSURE", None, "UNSURE"),
                                               ("DIFFERENT", None, None), ("NO_ANSWER", None, "NO_ANSWER")])
def test_gemini_verdict_decides_link_review_or_nothing(verdict, rule, item):
    nodes, res, series, links, review = _run(_VERDICT_METRICS, judge=VerdictJudge(verdict))
    assert [l["rule"] for l in links] == ([rule] if rule else [])
    amb = [i["payload"] for i in res["review_items"] if i["kind"] == "AMBIGUOUS_LINK"]
    assert [p["verdict"] for p in amb] == ([item] if item else [])


def test_a_missing_answer_is_not_a_no_and_shows_the_candidates():
    nodes, res, *_ = _run(_VERDICT_METRICS, judge=VerdictJudge("NO_ANSWER"))
    (item,) = [i for i in res["review_items"] if i["kind"] == "AMBIGUOUS_LINK"]
    assert item["payload"]["candidates"] == ["Tax Deducted at Source"] and item["payload"]["a_label_key"] == "advance income tax"


# ---------------------------------------------------------------- suspicious name links (M7)
def _name_pair(v_old, v_new, stmt="BALANCE_SHEET", cat_a="Current Liabilities", cat_b="Current Liabilities", field="Trade Payables"):
    return [_m("2019-20", field, {2020: v_old}, category=cat_a, stmt=stmt),
            _m("2020-21", field, {2021: 9_999_000, 2020: v_new}, category=cat_b, stmt=stmt)]


def test_small_restatement_on_a_name_link_stays_silent():
    nodes, res, series, links, review = _run(_name_pair(100_000_000, 103_000_000))
    assert [(l["rule"], l["confidence"]) for l in links] == [("NAME_EXACT", 1.0)]
    assert not [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]


def test_large_change_on_a_name_link_is_kept_but_flagged():
    # a HIGH variance (more than 100 % of the older figure) is shown for review
    nodes, res, series, links, review = _run(_name_pair(100_000_000, 250_000_000))
    assert [(l["rule"], l["confidence"]) for l in links] == [("NAME_EXACT", 0.8)]
    (item,) = [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]
    assert item["payload"]["rule"] == "NAME_EXACT" and "60 % in 2020" in item["payload"]["reason"]
    (conflict,) = sl.value_conflicts_from_review(res["review_items"])
    assert conflict["rationale"].startswith("Linked by name")
    # a big fall is a MID restatement (60 % of the older figure): the link keeps its doubt, but no review item is made
    nodes, res, series, links, review = _run(_name_pair(100_000_000, 40_000_000))
    assert [(l["rule"], l["confidence"]) for l in links] == [("NAME_EXACT", 0.8)]
    assert not [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]


def test_sign_flip_is_flagged_on_a_balance_line_only():
    nodes, res, *_ = _run(_name_pair(100_000_000, -90_000_000))
    (item,) = [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]
    assert "sign flip in 2020" in item["payload"]["reason"]
    nodes, res, *_ = _run(_name_pair(100_000_000, -90_000_000, stmt="CASH_FLOW", cat_a="Operating", cat_b="Operating",
                                     field="Trade Payables Increase"))
    reasons = [i["payload"]["reason"] for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]
    assert all("sign flip" not in r for r in reasons)


def test_name_link_under_unrelated_headings_is_flagged():
    # a 10 % restatement alone stays silent, but under headings with nothing in common it is shown
    nodes, res, *_ = _run(_name_pair(100_000_000, 110_000_000, cat_a="Related party transactions", cat_b="Other income statement"))
    (item,) = [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]
    assert item["payload"]["reason"].startswith("headings share no word")
    # agreeing values prove the link even under different headings
    nodes, res, *_ = _run(_name_pair(100_000_000, 100_000_000, cat_a="Related party transactions", cat_b="Other income statement"))
    assert not [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]


def test_exact_label_is_linked_before_a_looser_match_takes_the_node():
    # "Accounts Payable Total" would reach "accounts payable" through the "total" core match and comes first in label
    # order; the exact "Accounts Payable" head must still get it.
    loose = _m("2019-20", "Accounts Payable Total", {2020: 5_000_000}, category="Payables A", row=1)
    exact = _m("2019-20", "Accounts Payable", {2020: 5_000_000}, category="Payables A", row=2)
    new = _m("2020-21", "Accounts Payable", {2021: 6_000_000, 2020: 5_000_000}, category="Payables A", row=1)
    nodes = sl.build_nodes([loose, exact, new], {})
    heads = [n for n in nodes if n["report"] == "2019-20"]
    heads.sort(key=lambda n: n["label"], reverse=True)          # loose ("... total") first
    st = {"ov": sl._Overrides([], heads[0]["partition"]), "coexist": lambda a, b: False}
    res = sl._link_pair(heads[0]["partition"], st, heads, [n for n in nodes if n["report"] == "2020-21"])
    assert [(l["from_label"], l["to_label"]) for l in res["links"]] == [("account payable", "account payable")]


# ---------------------------------------------------------------- calculation rows (M14)
def test_sign_convention_change_is_flagged_on_the_link_and_the_merged_row():
    metrics = [_m("2022-23", "Energy Purchase", {2023: 62_703_036_003, 2022: 51_045_203_902}, category="Cost", stmt="INCOME_STATEMENT"),
               _m("2023-24", "Energy Purchase", {2024: -66_923_999_905, 2023: -62_703_036_003}, category="Cost", stmt="INCOME_STATEMENT")]
    nodes, res, series, links, review = _run(metrics)
    assert [l["rule"] for l in links] == ["NAME_EXACT"] and links[0]["confidence"] == 1.0
    assert links[0]["evidence"]["sign_convention_changed"] == [2023]
    assert not [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]       # same size: no review item
    out = sl.series_to_rows(series, [1], [], [])
    (row,) = out["rows"]
    assert row["sign_convention_years"] == ["2023"] and "SIGN_CONVENTION_CHANGED" in row["review_flags"]


def test_a_link_without_a_sign_flip_has_no_sign_flag():
    nodes, res, series, links, review = _run(_name_pair(100_000_000, 103_000_000))
    assert "sign_convention_changed" not in links[0]["evidence"]
    (row,) = sl.series_to_rows(series, [1], [], [])["rows"]
    assert row["sign_convention_years"] == [] and "SIGN_CONVENTION_CHANGED" not in row["review_flags"]


def test_exact_value_match_beats_a_doubtful_same_name():
    # 2018-19 "Operating Expenses" (298.5M) became "Direct Operating Expenses" in 2019-20; a NEW "Operating Expenses" group (2.27bn) took its name
    old = [_m("2018-19", "Operating Expenses", {2019: 298_504_417, 2018: 296_759_454}, category="Cost", stmt="INCOME_STATEMENT", row=1)]
    new = [_m("2019-20", "Operating Expenses", {2020: 2_545_036_938, 2019: 2_270_679_161}, category="Cost", stmt="INCOME_STATEMENT", row=1),
           _m("2019-20", "Direct Operating Expenses", {2020: 248_238_117, 2019: 298_504_417}, category="Cost", stmt="INCOME_STATEMENT", row=2)]
    # both names are printed together in a later report, which is what blocks a plain name / coexistence match
    later = [_m("2020-21", "Operating Expenses", {2021: 2_972_738_088, 2020: 2_545_036_938}, category="Cost", stmt="INCOME_STATEMENT", row=1),
             _m("2020-21", "Direct Operating Expenses", {2021: 837_282_121, 2020: 248_238_117}, category="Cost", stmt="INCOME_STATEMENT", row=2)]
    nodes, res, series, links, review = _run(old + new + later)
    first = [l for l in links if l["from_report"] == "2018-19"]
    assert [(l["rule"], l["to_label"]) for l in first] == [("VALUE_UNIQUE", "direct operating expense")]
    assert first[0]["evidence"]["preferred_over_name"] == "operating expense"
    (item,) = [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT" and i["payload"].get("rejected")]
    assert item["payload"]["rule"] == "NAME_EXACT" and "match 'Direct Operating Expenses' exactly" in item["payload"]["reason"]
    (conflict,) = [c for c in sl.value_conflicts_from_review(res["review_items"]) if not c["linked"]]
    assert conflict["rationale"].startswith("Same name, but the values match another line")


def test_a_big_change_without_an_exact_alternative_keeps_the_name_link():
    nodes, res, series, links, review = _run(_name_pair(100_000_000, 40_000_000))
    assert [(l["rule"], l["confidence"]) for l in links] == [("NAME_EXACT", 0.8)]


def test_definition_change_is_added_to_the_name_link_item():
    from app.ocr.services import series_merge
    metrics = _name_pair(100_000_000, 250_000_000, field="Cost of Sales")
    nodes, res, *_ = _run(metrics)
    (item,) = [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]
    a, b = (next(n for n in nodes if n["report"] == r) for r in ("2019-20", "2020-21"))
    cell = lambda n: next(iter(n["cells"]))        # noqa: E731
    comp = lambda n, parts: {"fiscal_year": n["report"], "page_number": cell(n)[0], "table_num": cell(n)[1],   # noqa: E731
                             "total": {"row": cell(n)[2], "col": cell(n)[3]}, "parts": [{"field": x} for x in parts]}
    series_merge._attach_definition_changes(res["review_items"], nodes, [comp(a, ["Energy Purchase", "Operating Expenses"]), comp(b, ["Direct Operating Expenses"])])
    assert item["payload"]["definition_change"] == {"a_parts": ["Energy Purchase", "Operating Expenses"], "b_parts": ["Direct Operating Expenses"]}
    assert item["payload"]["reason"].endswith("definition changed: Energy Purchase + Operating Expenses -> Direct Operating Expenses")
    (conflict,) = sl.value_conflicts_from_review(res["review_items"])
    assert conflict["definition_change"]["b_parts"] == ["Direct Operating Expenses"]


# ---------------------------------------------------------------- attention levels, soft review for Gemini restatements
def _gemini_pair(old_value, new_comparative):
    return [_m("2019-20", "Provision for the Year", {2020: old_value}, category="Tax"),
            _m("2020-21", "Provision during the Year", {2021: 9_000_000, 2020: new_comparative}, category="Tax")]


def test_gemini_link_with_a_smaller_difference_needs_no_review_item():
    # a restatement of any size up to 100 % is a type of the cell (RESTATED_LOW / MID), not a review item
    for comparative in (465_372_441, 700_000_000, 900_000_000):
        nodes, res, series, links, review = _run(_gemini_pair(465_332_031, comparative), judge=VerdictJudge("SAME"))
        assert [l["rule"] for l in links] == ["GEMINI"]
        assert [i["kind"] for i in res["review_items"]] == []
        assert sl.value_conflicts_from_review(res["review_items"]) == []        # no value mismatch on the merged row


def test_gemini_link_with_a_high_difference_stays_a_hard_value_conflict():
    nodes, res, series, links, review = _run(_gemini_pair(100_000_000, 465_332_031), judge=VerdictJudge("SAME"))
    assert [l["rule"] for l in links] == ["GEMINI"]
    (item,) = [i for i in res["review_items"] if i["kind"] in ("VALUE_CONFLICT", "RESTATEMENT_CHECK")]
    assert item["kind"] == "VALUE_CONFLICT" and sl.review_attention("VALUE_CONFLICT", item["payload"]) == "HIGH"


def test_unsure_gemini_answer_with_a_small_difference_is_an_ambiguous_link():
    nodes, res, series, links, review = _run(_gemini_pair(465_332_031, 465_372_441), judge=VerdictJudge("UNSURE"))
    assert links == []
    kinds = [i["kind"] for i in res["review_items"]]
    assert kinds.count("AMBIGUOUS_LINK") == 1 and "VALUE_CONFLICT" not in kinds


def test_attention_levels():
    big = {"evidence": {"overlap": [{"year": 2020, "a": 100.0, "b": 40.0, "match": False}]}, "statement_type": "INCOME_STATEMENT"}
    small = {"evidence": {"overlap": [{"year": 2020, "a": 100.0, "b": 90.0, "match": False}]}, "statement_type": "INCOME_STATEMENT"}
    flip = {"evidence": {"overlap": [{"year": 2020, "a": 100.0, "b": -90.0, "match": False}]}, "statement_type": "BALANCE_SHEET"}
    assert sl.review_attention("VALUE_CONFLICT", big) == "HIGH"
    assert sl.review_attention("VALUE_CONFLICT", flip) == "HIGH"
    assert sl.review_attention("VALUE_CONFLICT", {**small, "rejected": True}) == "HIGH"
    assert sl.review_attention("VALUE_CONFLICT", small) == "MEDIUM"            # e.g. only the headings differ
    assert sl.review_attention("MAGNITUDE_JUMP", {}) == "HIGH" and sl.review_attention("INTRA_NODE_CONFLICT", {}) == "HIGH"
    assert sl.review_attention("AMBIGUOUS_LINK", {}) == "MEDIUM" and sl.review_attention("SPLIT_DETECTED", {}) == "MEDIUM"
    assert sl.review_attention("RESTATEMENT_CHECK", {}) == "LOW" and sl.review_attention("SOMETHING_NEW", {}) == "MEDIUM"


def test_review_list_has_attention_and_is_sorted_most_important_first(ocr, monkeypatch):  # noqa: F811
    _seed("SOFTCO", "2019-20", 5, [_item("Provision for the Year", [(2020, 100_000_000)])])
    _seed("SOFTCO", "2020-21", 6, [{**_item("Provision during the Year", [(2021, 9_000_000), (2020, 465_372_441)]),
                                    "concept": {"name": "Provision during the Year", "statement": "BALANCE_SHEET", "category": "Other"}}])
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: VerdictJudge("SAME"))
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "SOFTCO"})
    items = ocr.get("/ocr/company/SOFTCO/merge/review", headers=KEY).json()["items"]
    (soft,) = [i for i in items if i["kind"] == "VALUE_CONFLICT"]
    assert soft["attention"] == "HIGH" and soft["attention_rank"] == 1
    assert [i["attention_rank"] for i in items] == sorted(i["attention_rank"] for i in items)
    r = ocr.post(f"/ocr/company/SOFTCO/merge/review/{soft['id']}/resolve", headers=KEY, json={"action": "accept"}).json()
    assert r["status"] == "resolved" and r["override_id"]                      # a link kind: accept saves a FORCE_LINK


# ---------------------------------------------------------------- restatement history (M8)
def _three_reports(v1, v2, v3):
    return [_m("2018-19", "Revenue Total", {2019: v1}, category="Revenue", stmt="INCOME_STATEMENT"),
            _m("2019-20", "Revenue Total", {2020: 9_000, 2019: v2}, category="Revenue", stmt="INCOME_STATEMENT"),
            _m("2020-21", "Revenue Total", {2021: 9_500, 2020: 9_000, 2019: v3}, category="Revenue", stmt="INCOME_STATEMENT")]


def _resolved(metrics):
    nodes, res, series, links, review = _run(metrics)
    (s,) = series
    return sl.resolve_series_values(s)


def test_a_figure_that_changes_in_a_middle_report_and_goes_back_is_still_a_restatement():
    r = _resolved(_three_reports(1_000_000, 1_400_000, 1_000_000))
    v = r["restate"]["2019"]
    assert v["original_value"] == v["restated_value"] == 1_000_000 and v["difference"] == 0
    assert [(h["from_report"], h["to_report"], h["percent_change"]) for h in v["history"]] == \
        [("2018-19", "2019-20", 40.0), ("2019-20", "2020-21", -28.57)]


def test_a_two_step_restatement_lists_every_step():
    r = _resolved(_three_reports(1_000_000, 1_400_000, 1_600_000))
    v = r["restate"]["2019"]
    assert v["original_value"] == 1_000_000 and v["restated_value"] == 1_600_000 and v["percent_change"] == 60.0
    assert len(v["history"]) == 2 and not any(h["within_tolerance"] for h in v["history"])


def test_rounding_drift_that_comes_back_shows_nothing_and_a_one_way_drift_is_a_micro_variance():
    r = _resolved(_three_reports(1_000_000_000, 1_000_000_001, 1_000_000_000))
    assert r["restate"] == {} and r["micro"] == {}
    r = _resolved(_three_reports(1_000_000_000, 1_000_000_001, 1_000_000_002))
    assert r["restate"] == {} and list(r["micro"]) == ["2019"]


def test_zero_base_gives_no_percent_instead_of_zero():
    r = _resolved(_three_reports(0, 500_000, 500_000))
    v = r["restate"]["2019"]
    assert v["original_value"] == 0 and v["percent_change"] is None
    assert v["history"][0]["percent_change"] is None


# ---------------------------------------------------------------- merge progress: Gemini calls
def _call_event(stage, call, calls, done_q, total_q, matched=0, unmatched=0, seconds=4.0):
    e = {"type": "progress", "stage": stage, "call_in_round": call, "calls_in_round": calls, "evaluated_pairs": done_q,
         "total_pairs": total_q, "matched_pairs": matched, "conflicts_count": unmatched, "percent": 0, "subtitle": "x"}
    if stage == "gemini_call_done":
        e["call_duration_sec"] = seconds
    return e


def test_plan_counts_the_rounds_that_ask_gemini_and_changes_nothing():
    import copy
    import json
    nodes = sl.build_nodes(_gemini_pair(465_332_031, 465_372_441), {})
    before = json.dumps(nodes, sort_keys=True, default=str)
    plan = series_merge.plan_gemini_calls(nodes, [])
    assert plan == [{"round": "2020-21", "questions": 1, "calls": 1}]
    assert json.dumps(nodes, sort_keys=True, default=str) == before            # the dry run leaves the nodes alone
    assert series_merge.plan_gemini_calls(sl.build_nodes([_m("2019-20", "Cash", {2020: 5})], {}), []) == []


def test_progress_turns_round_numbers_into_whole_merge_progress():
    events = []
    plan = [{"round": "2019-20", "questions": 4, "calls": 1}, {"round": "2021-22", "questions": 60, "calls": 2},
            {"round": "JOIN", "questions": 6, "calls": 1}]
    p = series_merge.GeminiProgress(events.append, plan)
    f = p.fields()
    assert (f["gemini_calls_total"], f["gemini_calls_done"], f["estimate_exact"], f["questions_total"], f["rounds_total"]) == (4, 0, False, 70, 3)
    assert f["eta_sec"] is None and f["round"] is None

    p.round_started("2019-20", 4)
    p.on_event(_call_event("gemini_call_started", 1, 1, 0, 4))
    p.on_event(_call_event("gemini_call_done", 1, 1, 4, 4, matched=3, unmatched=1))
    # the next round turns out to need 3 calls (150 questions) instead of 2: the total is corrected when it starts
    p.round_started("2021-22", 150)
    p.on_event(_call_event("gemini_call_started", 1, 3, 0, 150))
    p.on_event(_call_event("gemini_call_done", 1, 3, 50, 150, matched=40, unmatched=10, seconds=6.0))
    p.on_event(_call_event("gemini_retry", 2, 3, 50, 150, matched=40, unmatched=10))
    p.on_event(_call_event("gemini_call_done", 2, 3, 100, 150, matched=70, unmatched=30))
    p.on_event(_call_event("gemini_call_started", 3, 3, 100, 150, matched=70, unmatched=30))
    p.on_event(_call_event("gemini_call_done", 3, 3, 150, 150, matched=100, unmatched=50))
    p.round_started("JOIN", 6)                       # planned with 1 call, and it is 1 call
    p.on_event(_call_event("gemini_call_started", 1, 1, 0, 6))

    assert [e["gemini_calls_total"] for e in events] == [4, 4, 5, 5, 5, 5, 5, 5, 5]
    assert [e["gemini_calls_done"] for e in events] == [0, 1, 1, 2, 2, 3, 3, 4, 4]
    pct = [e["percent"] for e in events]
    assert pct == sorted(pct) and pct[0] == 10 and max(pct) <= 90
    last = events[-1]
    assert last["round"] == "JOIN" and last["round_index"] == 3 and last["rounds_total"] == 3
    assert last["total_pairs"] == 4 + 150 + 6 and last["evaluated_pairs"] == 4 + 150
    assert last["matched_pairs"] == 3 + 100 and last["conflicts_count"] == 1 + 50
    assert last["estimate_exact"] is False
    assert last["eta_sec"] == round((4.0 + 6.0 + 4.0 + 4.0) / 4 * 1, 1)           # mean call time x calls left
    assert events[2]["subtitle"] == "Asking Fintra AI (call 2 of about 5)"


def test_progress_without_a_plan_learns_the_rounds_as_they_come():
    events = []
    p = series_merge.GeminiProgress(events.append, [])
    assert p.fields()["gemini_calls_total"] == 0
    p.round_started("2020-21", 2)
    p.on_event(_call_event("gemini_call_started", 1, 1, 0, 2))
    assert events[0]["gemini_calls_total"] == 1 and events[0]["percent"] == 10
    p.on_event(_call_event("gemini_call_done", 1, 1, 2, 2))
    assert events[1]["gemini_calls_done"] == 1 and events[1]["percent"] == 90
    p.round_started("", 0)                                    # what the merge does before saving
    f = p.fields(exact=True)
    assert f["gemini_calls_done"] == 1 and f["estimate_exact"] is True


# ---------------------------------------------------------------- value conflicts carry both whole rows
def test_value_conflict_has_both_whole_rows_side_by_side():
    nodes, res, *_ = _run(_gemini_pair(100_000_000, 465_332_031), judge=VerdictJudge("SAME"))
    series_merge._attach_comparisons(res["review_items"], nodes)
    (item,) = [i for i in res["review_items"] if i["kind"] == "VALUE_CONFLICT"]
    cmp = item["payload"]["comparison"]
    assert cmp["years"] == [2020, 2021] and cmp["conflicting_years"] == ["2020"]
    assert cmp["year_status"] == {"2020": "MISMATCH", "2021": "ONLY_B"}
    a, b = cmp["lines"]
    assert (a["side"], a["report"], a["field"]) == ("a", "2019-20", "Provision for the Year")
    assert (b["side"], b["report"], b["field"]) == ("b", "2020-21", "Provision during the Year")
    assert a["values"] == {"2020": 100_000_000.0} and b["values"] == {"2020": 465_332_031.0, "2021": 9_000_000.0}
    assert b["cells"]["2021"] == {"page_number": 1, "table_num": 1, "table_row": 1, "table_col": 3}
    (conflict,) = sl.value_conflicts_from_review(res["review_items"])
    assert conflict["comparison"] == cmp and conflict["conflicting_years"] == ["2020"]


def test_value_conflict_without_node_ids_has_no_comparison():
    review = [{"kind": "VALUE_CONFLICT", "dedupe_key": "x", "payload": {"partition": "p", "evidence": {"overlap": []}}}]
    series_merge._attach_comparisons(review, [])
    assert "comparison" not in review[0]["payload"]


# ---------------------------------------------------------------- raw and merged tables as two endpoints
def test_raw_and_merged_results_are_separate_endpoints_with_the_shared_fields(ocr):  # noqa: F811
    def item(values):
        return {"concept": {"name": "Weighted average number of shares", "statement": "NOTES_AND_FOOTNOTES", "category": "EPS"},
                "scope": {"type": "STANDALONE"}, "raw_field": "Weighted average number of shares",
                "data": [{"year": y, "numeric_value": v, "unit": "Nos.", "multiplier": 1.0, "table_num": 1,
                          "table_row": 1, "table_col": 3 + i} for i, (y, v) in enumerate(values)]}
    for i, fy in enumerate(("2014-15", "2015-16")):
        database.upsert_page(None, "SPLTCO", fy, 10 + i, "", is_selected=True)
        database.update_page_ocr("SPLTCO", fy, 10 + i, MD)
    database.save_extracted_metrics("SPLTCO", "2014-15", 10, [item([(2015, 5), (2014, 4)])])
    database.save_extracted_metrics("SPLTCO", "2015-16", 11, [item([(2016, 6), (2015, 5)])])
    assert ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "SPLTCO"}).status_code == 200

    raw = ocr.get("/ocr/company/SPLTCO/results/raw", headers=KEY).json()
    merged = ocr.get("/ocr/company/SPLTCO/results/merged", headers=KEY).json()
    raw_only = {"raw_columns", "raw_csv_text", "raw_rows"}      # raw_csv_text is no longer in the raw response
    merged_only = {"merged_csv_text", "merged_rows", "value_mismatch_conflicts", "value_mismatch_count", "merge_report", "inactive_series_count"}
    common = {"instrument_code", "analyzed_years", "years", "has_raw_csv", "has_merged_csv", "merged_stale", "updated_at"}
    paging = {"statement_types", "total_rows", "limit", "offset", "has_more", "next_offset"}
    chunk = {"statement_type", "statement_types", "has_more", "next_statement_type"}
    assert set(raw) == common | (raw_only - {"raw_csv_text"}) | paging
    full = ocr.get("/ocr/company/SPLTCO/results/merged", headers=KEY, params={"view": "full"}).json()
    assert set(full) == common | (merged_only - {"merged_csv_text"}) | chunk
    # the default is the slim table: no merge report or conflict lists (counts instead), small rows
    assert set(merged) == common | {"merged_rows", "value_mismatch_count", "inactive_series_count", "value_conflicts_count",
                                    "value_merge_count"} | chunk
    assert set(merged["merged_rows"][0]) == {"series_id", "field", "scope", "statement_type", "unit", "metric_type", "level", "active",
                                             "values", "flags", "breakdown", "breakdown_of", "merged_from_count", "mismatch_years"}
    for k in common:
        assert raw[k] == merged[k], k
    assert raw["raw_rows"] and merged["merged_rows"] and full["merged_rows"][0]["restatement_dots"] is not None
    assert ocr.get("/ocr/company/NOSUCHCO/results/raw", headers=KEY).status_code == 404
    assert ocr.get("/ocr/company/NOSUCHCO/results/merged", headers=KEY).status_code == 404


def test_raw_rows_come_in_chunks_of_100_and_can_be_filtered_by_statement(ocr):  # noqa: F811
    def item(field, stmt, row, values):
        return {"concept": {"name": field, "statement": stmt}, "scope": {"type": "CONSOLIDATED"}, "raw_field": field,
                "data": [{"year": y, "numeric_value": v, "unit": "BDT", "multiplier": 1.0, "table_num": 1,
                          "table_row": row, "table_col": 3 + i} for i, (y, v) in enumerate(values)]}
    database.upsert_page(None, "PAGECO2", "2024-25", 1, "", is_selected=True)
    database.update_page_ocr("PAGECO2", "2024-25", 1, MD)
    items = ([item(f"Asset {i:03d}", "BALANCE_SHEET", i + 1, [(2025, 10 + i)]) for i in range(230)]
             + [item(f"Income {i:03d}", "INCOME_STATEMENT", i + 1, [(2025, 500 + i)]) for i in range(30)])
    database.save_extracted_metrics("PAGECO2", "2024-25", 1, items)
    get = lambda **params: ocr.get("/ocr/company/PAGECO2/results/raw", headers=KEY, params=params).json()   # noqa: E731

    c1 = get()
    c2, c3 = get(offset=c1["next_offset"]), get(offset=200)
    assert [len(c["raw_rows"]) for c in (c1, c2, c3)] == [100, 100, 60]
    assert (c1["next_offset"], c2["next_offset"], c3["next_offset"]) == (100, 200, None)
    assert (c1["has_more"], c3["has_more"], c1["total_rows"], c1["limit"], c1["offset"]) == (True, False, 260, 100, 0)
    assert "raw_csv_text" not in c1 and c1["has_raw_csv"] is True
    fields = [r["field"] for c in (c1, c2, c3) for r in c["raw_rows"]]
    assert len(set(fields)) == 260                                       # every row once, none repeated across chunks
    assert [(t["statement_type"], t["rows"]) for t in c1["statement_types"]] == [("BALANCE_SHEET", 230), ("INCOME_STATEMENT", 30)]
    assert c2["statement_types"] == c1["statement_types"] and c2["raw_columns"] == c1["raw_columns"] and c2["years"] == c1["years"]
    assert all("ref" not in c for r in c1["raw_rows"] for c in r["cells"].values())

    income = get(statement_type="income_statement")
    assert income["total_rows"] == 30 and len(income["raw_rows"]) == 30 and income["has_more"] is False
    assert {r["statement_type"] for r in income["raw_rows"]} == {"INCOME_STATEMENT"}
    assets = get(statement_type="BALANCE_SHEET", offset=200)
    assert (len(assets["raw_rows"]), assets["total_rows"], assets["next_offset"]) == (30, 230, None)
    assert get(statement_type="CASH_FLOW")["raw_rows"] == [] and get(offset=1000)["raw_rows"] == []


def test_merged_table_comes_one_statement_at_a_time_skipping_missing_statements(ocr):  # noqa: F811
    def item(field, stmt, scope, row):
        return {"concept": {"name": field, "statement": stmt}, "scope": {"type": scope}, "raw_field": field,
                "data": [{"year": y, "numeric_value": v, "unit": "BDT", "multiplier": 1.0, "table_num": 1,
                          "table_row": row, "table_col": 3 + i} for i, (y, v) in enumerate(((2025, 10 + row), (2024, 5 + row)))]}
    for i, fy in enumerate(("2023-24", "2024-25")):
        database.upsert_page(None, "CHNKCO", fy, 10 + i, "", is_selected=True)
        database.update_page_ocr("CHNKCO", fy, 10 + i, MD)
    # no BALANCE_SHEET at all: the first chunk must be the income statement
    items = ([item("Revenue", "INCOME_STATEMENT", "CONSOLIDATED", 1), item("Revenue", "INCOME_STATEMENT", "STANDALONE", 2),
              item("Cost", "INCOME_STATEMENT", "CONSOLIDATED", 3), item("Cash from operations", "CASH_FLOW", "CONSOLIDATED", 4),
              item("Other note", "NOTES_AND_FOOTNOTES", "CONSOLIDATED", 5)])
    database.save_extracted_metrics("CHNKCO", "2024-25", 11, items)
    database.save_extracted_metrics("CHNKCO", "2023-24", 10, [dict(it, data=[dict(d, year=d["year"] - 1) for d in it["data"]]) for it in items])
    assert ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "CHNKCO"}).status_code == 200
    get = lambda **params: ocr.get("/ocr/company/CHNKCO/results/merged", headers=KEY, params={"view": "full", **params}).json()   # noqa: E731

    first = get()
    assert first["statement_type"] == "INCOME_STATEMENT" and first["has_more"] is True
    assert first["next_statement_type"] == "CASH_FLOW"
    assert [t["statement_type"] for t in first["statement_types"]] == ["INCOME_STATEMENT", "CASH_FLOW", "NOTES_AND_FOOTNOTES"]
    assert first["statement_types"][0]["rows"] == len(first["merged_rows"]) >= 3
    assert {r["statement_type"] for r in first["merged_rows"]} == {"INCOME_STATEMENT"}
    assert {r["scope"] for r in first["merged_rows"]} == {"CONSOLIDATED", "STANDALONE"}      # both scopes in one chunk
    assert all("restatement_dots" in r for r in first["merged_rows"]) and "merged_csv_text" not in first
    assert {m["statement_type"] for m in first["merge_report"]} <= {"INCOME_STATEMENT"}
    second = get(statement_type=first["next_statement_type"])
    assert (second["statement_type"], second["next_statement_type"]) == ("CASH_FLOW", "NOTES_AND_FOOTNOTES")
    last = get(statement_type="notes_and_footnotes")
    assert (last["statement_type"], last["has_more"], last["next_statement_type"]) == ("NOTES_AND_FOOTNOTES", False, None)
    assert last["statement_types"] == first["statement_types"] and last["years"] == first["years"]
    missing = get(statement_type="BALANCE_SHEET")                                           # nothing there: point at the first existing one
    assert (missing["merged_rows"], missing["has_more"], missing["next_statement_type"]) == ([], True, "INCOME_STATEMENT")
    assert sum(len(c["merged_rows"]) for c in (first, second, last)) == sum(t["rows"] for t in first["statement_types"])

    database.upsert_page(None, "NOMRGCO", "2024-25", 1, "", is_selected=True)               # extracted, never merged
    database.update_page_ocr("NOMRGCO", "2024-25", 1, MD)
    database.save_extracted_metrics("NOMRGCO", "2024-25", 1, [item("Revenue", "INCOME_STATEMENT", "CONSOLIDATED", 1)])
    empty = ocr.get("/ocr/company/NOMRGCO/results/merged", headers=KEY).json()
    assert empty["merged_rows"] == [] and empty["statement_type"] is None and empty["statement_types"] == []
    assert empty["has_more"] is False and empty["next_statement_type"] is None


def test_breakdown_parts_and_totals_carry_their_series_and_cell_details_takes_a_metric_id(ocr):  # noqa: F811
    database.upsert_page(None, "BRK2CO", "2021-22", 7, "", is_selected=True)
    database.update_page_ocr("BRK2CO", "2021-22", 7, COMP_MD)
    database.save_extracted_metrics("BRK2CO", "2021-22", 7, ext.decode_compact_extraction(COMP_PAYLOAD, COMP_MD, report_end=2022))
    database.save_cell_compositions("BRK2CO", "2021-22", 7, ext.decode_compositions(COMP_PAYLOAD, COMP_MD))
    # before a merge nothing is in a series: null, and a metric id has none
    before = ocr.get("/ocr/company/BRK2CO/year/2021-22/compositions?page_number=7", headers=KEY).json()["compositions"]
    assert all(c["total"]["series_id"] is None for c in before)
    metric_id = next(p["metric_id"] for c in before for p in c["parts"] if p["metric_id"])
    assert ocr.get("/ocr/company/BRK2CO/cell-details", headers=KEY, params={"metric_id": metric_id, "year": 2022}).status_code == 404

    assert ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "BRK2CO"}).status_code == 200
    merged = {r["base_field"]: r["series_id"] for r in full_results(ocr, "BRK2CO")["merged_rows"]}
    comps = ocr.get("/ocr/company/BRK2CO/year/2021-22/compositions?page_number=7", headers=KEY).json()["compositions"]
    c = next(x for x in comps if x["total"]["raw_field"] == "Total balance with other banks")
    assert c["total"]["series_id"] == merged["Total balance with other banks"]
    assert {p["series_id"] for p in c["parts"]} == {merged["In Bangladesh"], merged["Outside Bangladesh"]}
    assert all(p["series_id"] for c in comps for p in c["parts"] if p["metric_id"])      # every part that is a line is in a series
    row = next(r for r in full_results(ocr, "BRK2CO")["merged_rows"] if r["base_field"] == "Total balance with other banks")
    printed = row["breakdown"]["by_year"]["2022"]                                         # the breakdown the 2021-22 report printed for 2022
    assert printed["report"] == "2021-22" and printed["status"] == "MATCH"
    assert {p["series_id"] for p in printed["components"]} == {merged["In Bangladesh"], merged["Outside Bangladesh"]}

    # the printed cells, one cell by position and a record's own breakdown carry it too
    cells = [x for v in ocr.get("/ocr/company/BRK2CO/extracted-cells", headers=KEY).json().values() for x in v]
    assert cells and all(x["series_id"] for x in cells if x["raw_field"] != "Net")
    assert all(x["series_id"] is None for x in cells if x["raw_field"] == "Net")      # a line the page does not name is in no series
    by_pos = ocr.get("/ocr/company/BRK2CO/cell-by-position", headers=KEY,
                     params={"fiscal_year": "2021-22", "page_number": 7, "table_num": 1, "table_row": c["total"]["row"],
                             "table_col": c["total"]["col"]}).json()
    assert by_pos["metric"]["series_id"] == merged["Total balance with other banks"]

    # cell-details by metric_id is the same as by its series_id; series_id wins when both are given
    part = next(p for p in c["parts"] if p["raw_field"] == "In Bangladesh")
    by_metric = ocr.get("/ocr/company/BRK2CO/cell-details", headers=KEY, params={"metric_id": part["metric_id"], "year": part["year"]}).json()
    by_series = ocr.get("/ocr/company/BRK2CO/cell-details", headers=KEY, params={"series_id": part["series_id"], "year": part["year"]}).json()
    assert by_metric["canonical_field"] == by_series["canonical_field"] and by_metric["cell_records"] == by_series["cell_records"]
    assert ocr.get("/ocr/company/BRK2CO/cell-details", headers=KEY, params={"year": 2022}).status_code == 422


def test_a_total_lists_every_part_any_report_printed_in_printed_order():
    """An older report prints a line (WPPF) the latest one dropped: it stays a component of the total, in its printed place."""
    def node(report, label, cells):
        return {"report": report, "fields": [label], "label": label, "cells": cells}

    def comp(fy, parts, total_row=9):
        return {"fiscal_year": fy, "page_number": 1, "table_num": 1, "direction": "UP", "operation": "SUM", "status": "MATCH",
                "total": {"row": total_row, "col": 3},
                "parts": [{"row": r, "col": 3, "sign": s} for r, s in parts]}

    old, new = "2022-23", "2023-24"
    series = [  # (total, loan, dividend, wppf, taxes), each printed in both reports (wppf only in the old one)
        {"nodes": [node(old, "Current Liabilities", {(1, 1, 9, 3)}), node(new, "Current Liabilities", {(1, 1, 9, 3)})]},
        {"nodes": [node(old, "Loan", {(1, 1, 1, 3)}), node(new, "Loan", {(1, 1, 1, 3)})]},
        {"nodes": [node(old, "Dividend", {(1, 1, 2, 3)}), node(new, "Dividend", {(1, 1, 2, 3)})]},
        {"nodes": [node(old, "WPPF", {(1, 1, 3, 3)})]},
        {"nodes": [node(old, "Taxes", {(1, 1, 4, 3)}), node(new, "Taxes", {(1, 1, 5, 3)})]},
    ]
    comps = [comp(old, [(1, 1), (2, 1), (3, 1), (4, 1)]), comp(new, [(1, 1), (2, 1), (5, -1)])]
    per_total, part_of = sl.series_breakdowns(series, [10, 11, 12, 13, 14], comps)
    bd = per_total[10]
    assert [c["series_id"] for c in bd["components"]] == [11, 12, 13, 14]                  # WPPF after Dividend
    assert [c["last_report"] for c in bd["components"]] == [new, new, old, new]
    assert bd["report"] == new and bd["components"][3]["sign"] == -1                        # latest report's sign
    assert part_of[13] == [10] and part_of[11] == [10]


def test_gap_items_carry_the_printed_cells_and_a_comparison_of_both_lines():
    nodes, res, _, _, _ = _run(_reports_with_filler(YEARS, {"2012-13", "2014-15", "2017-18"}))
    review = list(res["review_items"])
    series_merge._attach_comparisons(review, nodes)
    (item,) = [i for i in review if i["kind"] == "LONG_GAP"]
    p = item["payload"]
    assert p["a_report"] == "2014-15" and p["b_report"] == "2017-18"
    assert p["a_page_number"] is not None and p["b_table_col"] is not None
    cmp_ = p["comparison"]
    assert [l["side"] for l in cmp_["lines"]] == ["a", "b"]
    assert set(cmp_["year_status"]) == {str(y) for y in cmp_["years"]}
    assert set(cmp_["year_status"].values()) <= {"MATCH", "MISMATCH", "ONLY_A", "ONLY_B"}
    for line in cmp_["lines"]:
        assert set(line["cells"]) == set(line["values"])



# ---------------------------------------------------------------- review items by origin, LLM overrides
def _unid(field, row, flag="FIELD_UNIDENTIFIED"):
    return {**_item(field, [(2021, 5)]), "verification_flag": flag, "flag_reason": "unsure",
            "data": [{"year": 2021, "numeric_value": 5.0, "unit": "BDT", "multiplier": 1.0, "table_num": 1,
                      "table_row": row, "table_col": 3}]}


def _review(client, code, **params):
    return client.get(f"/ocr/company/{code}/merge/review", headers=KEY, params={"status": "all", **params}).json()["items"]


USER = {**KEY, "X-Username": "Fin001"}


def test_extraction_review_items_follow_their_page_and_survive_a_merge(ocr, monkeypatch):  # noqa: F811
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: RecordingJudge())
    _seed("XRCO", "2020-21", 5, [_unid("Total", 8), _item("Loans", [(2021, 7), (2020, 6)])])
    _seed("XRCO", "2020-21", 6, [_unid("Total", 9)])
    items = _review(ocr, "XRCO")
    assert [(i["kind"], i["origin"], i["page_number"]) for i in items] == [("FIELD_UNIDENTIFIED", "extraction", 5),
                                                                           ("FIELD_UNIDENTIFIED", "extraction", 6)]
    assert items[0]["payload"]["table_col"] == 3

    # dismissed by a person, then a merge: extraction items are not touched, and the merge makes none of them
    ocr.post(f"/ocr/company/XRCO/merge/review/{items[0]['id']}/resolve", headers=USER, json={"action": "dismiss"})
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "XRCO"})
    after = _review(ocr, "XRCO")
    assert {(i["kind"], i["page_number"], i["status"]) for i in after if i["origin"] == "extraction"} == \
        {("FIELD_UNIDENTIFIED", 5, "dismissed"), ("FIELD_UNIDENTIFIED", 6, "open")}
    assert not [i for i in after if i["kind"] == "FIELD_UNIDENTIFIED" and i["origin"] == "merge"]

    # page 5 is extracted again, now identified: its item is gone (even the dismissed one); page 6 keeps its own
    _seed("XRCO", "2020-21", 5, [_item("Loans", [(2021, 7), (2020, 6)])])
    assert [(i["page_number"], i["status"]) for i in _review(ocr, "XRCO") if i["origin"] == "extraction"] == [(6, "open")]


def test_naming_a_line_closes_its_extraction_review_item(ocr):  # noqa: F811
    _seed("XRCO2", "2020-21", 5, [_unid("Total", 8, flag="LABEL_EVIDENCE_FAILED")])
    (item,) = _review(ocr, "XRCO2")
    assert item["kind"] == "LABEL_EVIDENCE_FAILED" and item["payload"]["field"] == "Total"
    r = ocr.post("/ocr/company/XRCO2/line-overrides", headers=USER,
                 json={"fiscal_year": "2020-21", "page_number": 5, "table_row": 8, "field": "Other assets"})
    assert r.status_code == 200
    assert _review(ocr, "XRCO2") == []


def _tax_pair(code):
    _seed(code, "2019-20", 5, [_item("Advance Income Tax", [(2020, 160_281_671)])])
    _seed(code, "2020-21", 6, [{**_item("Tax Deducted at Source", [(2021, 170_000_000), (2020, 160_281_671)]),
                                "concept": {"name": "Tax Deducted at Source", "statement": "BALANCE_SHEET", "category": "Other"}}])


def test_merge_replaces_its_review_items(ocr, monkeypatch):  # noqa: F811
    _tax_pair("RVCO")
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: RecordingJudge(choose=0, conf=0.70))
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "RVCO"})
    (first,) = [i for i in _review(ocr, "RVCO") if i["kind"] == "AMBIGUOUS_LINK"]
    ocr.post(f"/ocr/company/RVCO/merge/review/{first['id']}/resolve", headers=USER, json={"action": "dismiss"})
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "RVCO"})
    (again,) = [i for i in _review(ocr, "RVCO") if i["kind"] == "AMBIGUOUS_LINK"]
    assert again["id"] != first["id"] and again["status"] == "open"       # replaced by the new merge, not kept as dismissed


def _pair_item(kind="LONG_GAP"):
    return {"kind": kind, "dedupe_key": "a>b", "payload": {"partition": "BALANCE_SHEET|STANDALONE||", "a_label_key": "advance income tax",
                                                           "a_report": "2019-20", "b_label_key": "tax deducted at source", "b_report": "2020-21"}}


def _ov(**kw):
    return {"id": 7, "kind": "FORCE_LINK", "statement_type": "BALANCE_SHEET", "scope": "STANDALONE", "entity_key": "", "segment_key": "",
            "a_label_key": "Advance Income Tax", "a_report": None, "b_label_key": "Tax Deducted at Source", "b_report": None,
            "source": "user", "active": 1, **kw}


def test_auto_resolve_marks_items_a_user_override_already_decides():
    for ov in (_ov(), _ov(kind="FORCE_SPLIT", a_label_key="Tax Deducted at Source", b_label_key="Advance Income Tax"),
               _ov(a_report="2019-20", b_report="2020-21"), _ov(kind="DETACH", b_label_key=None)):
        review = [_pair_item()]
        series_merge.auto_resolve(review, [ov])
        assert (review[0]["status"], review[0]["resolution"], review[0]["override_id"]) == ("resolved", "override", 7)
    for ov in (_ov(source="llm"), _ov(a_report="2018-19", b_report="2019-20"), _ov(a_label_key="Other"), _ov(active=0)):
        review = [_pair_item()]
        series_merge.auto_resolve(review, [ov])
        assert "status" not in review[0]
    orphan = [{"kind": "ORPHAN_OVERRIDE", "dedupe_key": "7", "payload": {}}]
    series_merge.auto_resolve(orphan, [_ov()])
    assert "status" not in orphan[0]


def _llm_company(ocr, monkeypatch, code):  # noqa: F811
    _tax_pair(code)
    judge = VerdictJudge("SAME")
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: judge)
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": code})
    return judge


def _overrides(ocr, code, **params):  # noqa: F811
    return ocr.get(f"/ocr/company/{code}/merge/overrides", headers=KEY, params=params).json()["overrides"]


def test_gemini_links_are_llm_overrides_that_every_merge_decides_again(ocr, monkeypatch):  # noqa: F811
    judge = _llm_company(ocr, monkeypatch, "LLCO")
    (o,) = _overrides(ocr, "LLCO", source="llm")
    assert (o["kind"], o["source"], o["link_rule"], o["a_report"], o["b_report"]) == ("FORCE_LINK", "llm", "GEMINI", "2019-20", "2020-21")
    assert o["confidence"] == 0.95 and _overrides(ocr, "LLCO", source="user") == []
    assert full_results(ocr, "LLCO")["merged_stale"] is False             # the overrides a merge wrote do not make that merge stale
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "LLCO"})
    assert len(judge.questions) == 2                                      # asked again, not read back from the override
    (o2,) = _overrides(ocr, "LLCO")
    assert o2["source"] == "llm" and o2["id"] != o["id"]


def test_make_permanent_keeps_the_link_and_stops_asking(ocr, monkeypatch):  # noqa: F811
    judge = _llm_company(ocr, monkeypatch, "LLCO2")
    (o,) = _overrides(ocr, "LLCO2")
    r = ocr.post(f"/ocr/company/LLCO2/merge/override/{o['id']}/decide", headers=USER, json={"action": "make_permanent"})
    assert r.status_code == 200 and r.json()["override"]["source"] == "user" and r.json()["merged_stale"] is True
    assert ocr.post(f"/ocr/company/LLCO2/merge/override/{o['id']}/decide", headers=USER,
                    json={"action": "break_link"}).status_code == 422       # no longer an LLM override
    rows = ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "LLCO2"}).json()["merged_csv_data"]["rows"]
    assert len(rows) == 1 and rows[0]["link_summary"][0]["rule"] == "USER" and len(judge.questions) == 1
    assert [x["id"] for x in _overrides(ocr, "LLCO2")] == [o["id"]]          # kept, not rewritten


def test_break_link_splits_the_lines_for_good(ocr, monkeypatch):  # noqa: F811
    _llm_company(ocr, monkeypatch, "LLCO3")
    (o,) = _overrides(ocr, "LLCO3")
    r = ocr.post(f"/ocr/company/LLCO3/merge/override/{o['id']}/decide", headers=USER, json={"action": "break_link"})
    assert r.status_code == 200 and r.json()["override"]["kind"] == "FORCE_SPLIT" and r.json()["override"]["source"] == "user"
    assert ocr.post("/ocr/company/LLCO3/merge/override/9999/decide", headers=USER, json={"action": "break_link"}).status_code == 404
    rows = ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "LLCO3"}).json()["merged_csv_data"]["rows"]
    assert sorted(r_["field"] for r_ in rows) == ["Advance Income Tax", "Tax Deducted at Source"]
    assert [x["source"] for x in _overrides(ocr, "LLCO3")] == ["user"]       # the LLM override is gone, the split stays


# ---------------------------------------------------------------- entity words: a question for Fintra AI, never an automatic link
_ENTITY_METRICS = [_m("2018-19", "Investment in Associate Companies", {2019: 256_342_357, 2018: 200_000_000}, category="Non-Current Assets"),
                   _m("2019-20", "Investment in Associate and Subsidiary Companies", {2020: 281_256_372, 2019: 256_342_357}, category="Non-Current Assets")]


def test_an_entity_word_difference_goes_to_fintra_ai_when_the_common_year_agrees():
    judge = RecordingJudge(choose=0)
    _, _, series, links, _ = _run(_ENTITY_METRICS, judge=judge)
    assert [len(q["candidates"]) for q in judge.questions] == [1]
    assert "Subsidiary" in judge.questions[0]["candidates"][0]["label"] and judge.questions[0]["candidates"][0]["evidence"].startswith("=2019")
    assert [l["rule"] for l in links] == ["GEMINI"]                                  # Fintra AI said SAME: one series
    assert _names(series) == [["Investment in Associate Companies", "Investment in Associate and Subsidiary Companies"]]


def test_an_entity_word_pair_is_never_linked_without_fintra_ai_saying_so():
    assert _run(_ENTITY_METRICS, judge=None)[3] == []                                # R2 (the identical figure) still refuses it
    assert _run(_ENTITY_METRICS, judge=RecordingJudge(choose=-1))[3] == []            # Fintra AI says no match
    assert _run(_ENTITY_METRICS, judge=VerdictJudge("DIFFERENT"))[3] == []


def test_an_entity_word_pair_across_a_gap_reaches_the_join_pass():
    ms = [_m("2018-19", "Investment in Associate Companies", {2019: 256_342_357, 2018: 200_000_000}, category="Non-Current Assets"),
          _m("2019-20", "Cash at Bank", {2020: 5_000_001, 2019: 4_000_001}, category="Current Assets"),
          _m("2020-21", "Cash at Bank", {2021: 6_000_003, 2020: 5_000_001}, category="Current Assets"),
          _m("2021-22", "Investment in Associate and Subsidiary Companies", {2022: 400_000_000, 2021: 331_045_029}, category="Non-Current Assets")]
    judge = RecordingJudge(choose=0)
    _, _, series, links, _ = _run(ms, judge=judge)
    assert any("Subsidiary" in c["label"] for q in judge.questions for c in q["candidates"])
    assert [l["rule"] for l in links if "associate" in l["from_label"]] == ["GEMINI_JOIN"]


def test_hard_vetoes_are_still_never_asked():
    judge = RecordingJudge(choose=0)
    assert _run(_ENTITY_METRICS, [_override("FORCE_SPLIT", "Investment in Associate Companies", "Investment in Associate and Subsidiary Companies")], judge=judge)[3] == []
    assert judge.questions == []
    judge = RecordingJudge(choose=0)                                                  # a subtotal is never its own component
    _run([_m("2019-20", "Income Tax Expense", {2020: 5_000_001, 2019: 4_000_001}), _m("2020-21", "Income Tax Expense Subtotal", {2021: 6_000_003, 2020: 5_000_001})], judge=judge)
    assert not any("Subtotal" in c["label"] for q in judge.questions for c in q["candidates"])


def test_subtotal_vs_total_with_the_same_other_words_is_no_contrast_but_a_part_vs_its_subtotal_is():
    from app.ocr.services.merge_service import has_aggregation_level_contrast as contrast
    for a, b in [("Cash Subtotal", "Cash Total"), ("Total Cash", "Cash Sub-total"),
                 ("Balance with other banks and financial institutions subtotal", "Balance with other banks and financial institutions total")]:
        assert contrast(a, b)[0] is False, (a, b)
    for a, b in [("Cash", "Cash Subtotal"), ("Income Tax Expense", "Income Tax Expense Subtotal"),
                 ("Cash Subtotal", "Deposits Total"), ("Government Investments", "Investments Subtotal")]:
        assert contrast(a, b) == (True, "Aggregation level contrast: subtotal vs component"), (a, b)


_CASH = {"2015-16": ("Cash Total", {2016: 24_484_298_511, 2015: 21_000_000_000}),
         "2016-17": ("Cash Subtotal", {2017: 23_787_103_908, 2016: 24_484_298_511}),      # printed "Subtotal" in this report only
         "2017-18": ("Cash Total", {2018: 19_440_423_683, 2017: 23_787_103_908})}


def test_a_line_printed_subtotal_in_one_report_and_total_in_the_others_is_one_series():
    metrics = [_m(fy, f, v, category="Assets") for fy, (f, v) in _CASH.items()]
    nodes, res, series, links, review = _run(metrics)
    assert len(series) == 1 and len(series[0]["nodes"]) == 3
    assert [l["rule"] for l in links] == ["VALUE_UNIQUE", "VALUE_UNIQUE"]


def test_subtotal_and_total_with_different_common_year_values_are_not_linked():
    changed = dict(_CASH)
    changed["2016-17"] = ("Cash Subtotal", {2017: 23_787_103_908, 2016: 30_000_000_000})
    metrics = [_m(fy, f, v, category="Assets") for fy, (f, v) in changed.items()]
    nodes, res, series, links, review = _run(metrics)
    assert len(series) > 1 and not any(l["rule"] == "VALUE_UNIQUE" and "Subtotal" in l["from_label"] + l["to_label"] for l in links)


def test_subtotal_and_total_printed_together_in_one_report_stay_two_series():
    metrics = [_m("2020-21", "Cash Total", {2021: 100.0}, category="Assets", row=1),
               _m("2020-21", "Cash Subtotal", {2021: 250.0}, category="Assets", row=2),
               _m("2021-22", "Cash Total", {2022: 120.0, 2021: 100.0}, category="Assets", row=1),
               _m("2021-22", "Cash Subtotal", {2022: 300.0, 2021: 250.0}, category="Assets", row=2)]
    nodes, res, series, links, review = _run(metrics)
    assert sorted(len(s["nodes"]) for s in series) == [2, 2]


def test_the_link_prompt_tells_fintra_ai_how_to_treat_entity_words_and_standard_totals():
    from app.ocr.services import merge_service
    p = merge_service.LINK_SYSTEM_INSTRUCTION
    assert "Entity words" in p and "parent" in p and "Standard totals" in p and "differently NAMED subsidiaries" in p
    assert "Net Cash Increase" in p and "A subtotal is never the same line" in p


def test_user_line_names_survive_reextraction_and_remerge(ocr, monkeypatch):  # noqa: F811
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: RecordingJudge())
    _seed("KPCO", "2020-21", 5, [_unid("Total", 8)])
    ocr.post("/ocr/company/KPCO/line-overrides", headers=USER,
             json={"fiscal_year": "2020-21", "page_number": 5, "table_row": 8, "field": "Other assets"})
    _seed("KPCO", "2020-21", 5, [_unid("Total", 8)])                      # extracted again: flagged again, then named again
    ocr.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": "KPCO"})
    assert [x["field"] for x in ocr.get("/ocr/company/KPCO/line-overrides", headers=KEY).json()] == ["Other assets"]
    assert _review(ocr, "KPCO") == []


def test_user_overrides_record_who_made_them(ocr, monkeypatch):  # noqa: F811
    _llm_company(ocr, monkeypatch, "WHCO")
    (llm,) = _overrides(ocr, "WHCO")
    assert llm["source"] == "llm" and llm["created_by"] is None          # decided by Gemini, no person
    r = ocr.post("/ocr/company/WHCO/merge/override", headers={**KEY, "X-Username": "Alice"}, json={
        "kind": "RENAME", "statement_type": "BALANCE_SHEET", "scope": "STANDALONE", "a_label": "Advance Income Tax", "new_label": "Tax"})
    assert r.status_code == 201
    done = ocr.post(f"/ocr/company/WHCO/merge/override/{llm['id']}/decide", headers={**KEY, "X-Username": "Bob"},
                    json={"action": "make_permanent"}).json()["override"]
    assert done["created_by"] == "Bob"
    by_id = {o["id"]: o["created_by"] for o in _overrides(ocr, "WHCO", source="user")}
    assert by_id == {r.json()["id"]: "Alice", llm["id"]: "Bob"}


def _socket_messages(ws):
    out = []
    while True:
        try:
            out.append(ws.receive_json())
        except Exception:                 # the server closed the socket after its last message
            return out


def test_merge_socket_feeds_the_progress_and_closes_when_the_merge_is_done(ocr, monkeypatch):  # noqa: F811
    _tax_pair("WSCO")
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: VerdictJudge("SAME"))
    with ocr.websocket_connect("/ocr/ws/merge/WSCO?username=Alice&api_key=k3y") as ws:
        msgs = _socket_messages(ws)
    types = [m["type"] for m in msgs]
    assert types[0] == "queued" and msgs[0]["job_id"]
    assert types[-1] == "complete" and types.count("complete") == 1 and msgs[-1]["percent"] == 100
    assert "init" in types and "finalize" in types and "error" not in types
    percents = [m["percent"] for m in msgs if "percent" in m]
    assert percents == sorted(percents)
    assert msgs[-1]["merged_rows_count"] == 1 and "merged_csv_data" not in msgs[-1]
    steps = [m for m in msgs if m.get("stage") == "report_linked"]
    assert [m["merge_stats"]["reports"]["done"] for m in steps][:2] == [1, 2]
    first = steps[0]["merge_stats"]
    assert first["lines"] == {"total": 2, "done": 1, "remaining": 1}          # one report of two is linked
    assert all("merge_stats" in m for m in msgs if m["type"] in ("init", "progress", "finalize"))
    final = msgs[-1]["merge_stats"]
    assert final["lines"] == {"total": 2, "done": 2, "remaining": 0} and final["reports"]["remaining"] == 0
    assert final["links"] == {"total": 1, "by_system": 0, "by_user": 0, "by_fintra_ai": 1}
    assert final["fintra_ai"]["questions_total"] == 1 and final["fintra_ai"]["linked"] == 1
    assert final["reviews"]["complete"] is True and final["reviews"]["created"] == final["reviews"]["high"] + final["reviews"]["medium"] + final["reviews"]["low"]
    assert [e["username"] for e in ocr.get("/ocr/events", headers=KEY, params={"instrument_code": "WSCO"}).json()["events"]
            if e["action"] == "MERGE_RUN"] == ["Alice"]
    assert full_results(ocr, "WSCO")["merged_stale"] is False             # the merge was saved


def _poll_messages(ocr, job_id, step=None):
    """GET /jobs/{id}/events until done, `after` = the previous `next`; every message once. step(i) runs between polls."""
    out, after, i = [], 0, 0
    while True:
        r = ocr.get(f"/ocr/jobs/{job_id}/events", headers=KEY, params={"after": after})
        assert r.status_code == 200
        body = r.json()
        out += body["messages"]
        assert body["next"] >= after
        after = body["next"]
        if body["done"]:
            again = ocr.get(f"/ocr/jobs/{job_id}/events", headers=KEY, params={"after": after}).json()
            assert again["messages"] == [] and again["next"] == after          # nothing repeated after the end
            return out
        i += 1
        assert i < 2000
        if step:
            step(i)


def test_merge_polling_gives_the_socket_messages(ocr, monkeypatch):  # noqa: F811
    import time
    _tax_pair("PLCO")
    monkeypatch.setattr(series_merge, "_default_judge", lambda *a, **k: VerdictJudge("SAME"))
    job = ocr.post("/ocr/jobs/merge", headers={**KEY, "X-Username": "Alice"}, json={"instrument_code": "PLCO"}).json()
    msgs = _poll_messages(ocr, job["job_id"], step=lambda i: time.sleep(0.01))
    types = [m["type"] for m in msgs]
    assert types[0] == "queued" and types[1] == "started" and types[-1] == "complete" and types.count("complete") == 1
    assert "init" in types and "finalize" in types and "error" not in types
    assert all("merged_csv_data" not in m for m in msgs)
    percents = [m["percent"] for m in msgs if "percent" in m]
    assert percents == sorted(percents) and msgs[-1]["percent"] == 100
    assert any(m.get("stage") == "report_linked" for m in msgs)
    assert all("merge_stats" in m for m in msgs if m["type"] in ("init", "progress", "finalize"))
    final = msgs[-1]["merge_stats"]
    assert final["links"]["by_fintra_ai"] == 1 and final["reviews"]["complete"] is True
    assert msgs[-1]["merged_rows_count"] == 1
    # the same messages from one read at the end
    whole = ocr.get(f"/ocr/jobs/{job['job_id']}/events", headers=KEY).json()
    assert whole["messages"] == msgs and whole["done"] is True
    assert ocr.get("/ocr/jobs/" + "0" * 32 + "/events", headers=KEY).status_code == 404


def test_merge_polling_reports_an_error_as_the_last_message(ocr):  # noqa: F811
    job = ocr.post("/ocr/jobs/merge", headers={**KEY, "X-Username": "Alice"}, json={"instrument_code": "NOSUCHCO"}).json()
    msgs = _poll_messages(ocr, job["job_id"])
    assert msgs[0]["type"] == "queued" and msgs[-1]["type"] == "error" and msgs[-1]["status_code"] == 404


@pytest.mark.parametrize("url,status", [("/ocr/ws/merge/WSCO2?username=Alice&api_key=wrong", 401),
                                        ("/ocr/ws/merge/WSCO2?api_key=k3y", 400),
                                        ("/ocr/ws/merge/WSCO2?username=Alice&api_key=k3y", 404)])
def test_merge_socket_refuses_with_one_error_message(ocr, url, status):  # noqa: F811
    with ocr.websocket_connect(url) as ws:
        msgs = _socket_messages(ws)
    assert [(m["type"], m["status_code"]) for m in msgs] == [("error", status)]


def test_notes_and_changes_in_equity_items_are_always_low():
    for stmt in ("NOTES_AND_FOOTNOTES", "CHANGES_IN_EQUITY"):
        for kind in ("MAGNITUDE_JUMP", "INTRA_NODE_CONFLICT", "AMBIGUOUS_LINK", "LONG_GAP", "FIELD_UNIDENTIFIED", "SPLIT_DETECTED"):
            assert sl.review_attention(kind, {"statement_type": stmt}) == "LOW"
            assert sl.review_attention(kind, {"partition": f"{stmt}|CONSOLIDATED||"}) == "LOW"     # items that carry only the partition
        flip = {"statement_type": stmt, "evidence": {"overlap": [{"year": 2021, "a": 5, "b": -5, "match": False}]}, "rejected": True}
        assert sl.review_attention("VALUE_CONFLICT", flip) == "LOW"
    assert sl.review_attention("MAGNITUDE_JUMP", {"statement_type": "BALANCE_SHEET"}) == "HIGH"
    assert sl.review_attention("INTRA_NODE_CONFLICT", {"partition": "CASH_FLOW|STANDALONE||"}) == "HIGH"


def test_year_breakdowns_keep_what_each_report_printed_and_only_when_it_adds_up():
    def node(report, label, cells):
        return {"report": report, "fields": [label], "label": label, "cells": cells}

    def comp(fy, parts, status="MATCH", total_row=9):
        return {"fiscal_year": fy, "page_number": 1, "table_num": 1, "direction": "UP", "operation": "SUM", "status": status,
                "total": {"row": total_row, "col": 3},
                "parts": [{"row": r, "col": 3, "sign": s, "value": v, "field": f"line {r}"} for r, s, v in parts]}

    old, new = "2022-23", "2023-24"
    series = [{"nodes": [node(old, "Total", {(1, 1, 9, 3)}), node(new, "Total", {(1, 1, 9, 3)})]},
              {"nodes": [node(old, "Loan", {(1, 1, 1, 3)}), node(new, "Loan", {(1, 1, 1, 3)})]}]
    comps = [comp(old, [(1, 1, 60.0), (2, 1, 5.0)]), comp(new, [(1, 1, 60.0)], status="MISMATCH")]
    out = sl.year_breakdowns(series, [10, 11], comps)
    assert list(out) == [(old, 1, 1, 9, 3), (new, 1, 1, 9, 3)]
    assert out[(new, 1, 1, 9, 3)] == {"report": new, "status": "MISMATCH"}              # stored but no basis for a year: status only, no parts
    assert out[(old, 1, 1, 9, 3)]["components"] == [
        {"series_id": 11, "sign": 1, "value": 60.0}, {"series_id": None, "sign": 1, "value": 5.0, "field": "line 2"}]
