"""Merge helpers shared by the series merge (app/ocr/services/series_merge.py):

- lexical helpers and hard accounting contrasts (vetoes) between two line-item labels;
- the Gemini judge used only for ambiguous cross-report links: for a line item from one report it
  picks which candidate line in the next report (if any) is the same item, seeing value evidence.
"""

import json
import logging
import re
import time
from difflib import SequenceMatcher
from typing import Any, Callable, Dict, List, Optional, Set, Tuple

from google import genai

from ..config import DEFAULT_MODEL, gemini_api_key
from ..rate_limiter import rate_limiters
from .gemini_json import generate_json, response_text, usage_tokens

logger = logging.getLogger("ocr.merge_service")


class MergeCancelled(Exception):
    """Raised when the caller asked a running merge to stop (e.g. the client disconnected)."""


def _check_cancelled(should_cancel: Optional[Callable[[], bool]]) -> None:
    if should_cancel is not None and should_cancel():
        raise MergeCancelled()

# ---------------------------------------------------------------------------
# Text normalization & lexical patterns
# ---------------------------------------------------------------------------

def normalize_text(s: str) -> str:
    cleaned = re.sub(r"[^a-zA-Z0-9\s]", " ", str(s).lower())
    return re.sub(r"\s+", " ", cleaned).strip()

def get_tokens(s: str) -> Set[str]:
    return set(re.findall(r"[a-zA-Z0-9]+", str(s).lower()))


SYNONYM_REPLACEMENTS = [
    (r"\bloan\b", "borrowings"),
    (r"\bloans\b", "borrowings"),
    (r"\bsales\b", "revenue"),
    (r"\bturnover\b", "revenue"),
    (r"\bobligation\b", "obligations"),
    (r"\bat\b", "of"),
    (r"\bincome-tax\b", "income tax"),
    (r"\b(net\s+assets?\s+value|nav)\s+per\s+shares?\b|\bnavps\b", "navps"),
    (r"\b(net\s+operating\s+)?cash\s+flow\s+per\s+shares?\b|\bnocfps\b", "nocfps"),
    (r"\bweighted\s+average\s+number\s+of\s+shares\b", "number of shares"),
    (r"\bshare\b", "shares"),
]

def apply_synonyms(s: str) -> str:
    res = str(s).lower()
    for pat, rep in SYNONYM_REPLACEMENTS:
        res = re.sub(pat, rep, res)
    return normalize_text(res)

CONTRAST_SPECS = [
    # (regex_a, label_a, regex_b, label_b)
    (r"\bnon\s*current\b|\blong\s*term\b|\bfixed\s+assets?\b", "non-current", r"\bcurrent\b", "current"),
    (r"\boperating\b", "operating", r"\bnon\s*operating\b", "non-operating"),
    (r"\boperating\b", "operating", r"\bfinancing\b", "financing"),
    (r"\boperating\b", "operating", r"\binvesting\b", "investing"),
    (r"\bfinancing\b", "financing", r"\binvesting\b", "investing"),
    (r"\bassets?\b", "asset", r"\bliabilit(y|ies)\b", "liability"),
    (r"\bassets?\b", "asset", r"\bequit(y|ies)\b", "equity"),
    (r"\bliabilit(y|ies)\b", "liability", r"\bequit(y|ies)\b", "equity"),
    (r"\b(in|inside)\s+bangladesh\b|\bdomestic\b|\blocal\b", "domestic/in-bangladesh", r"\boutside\s+bangladesh\b|\bforeign\b|\babroad\b", "foreign/outside-bangladesh"),
    (r"\bsecured\b", "secured", r"\bunsecured\b", "unsecured"),
    (r"\bdirect\b", "direct", r"\bindirect\b", "indirect"),
    (r"\bgross\b", "gross", r"\bnet\b", "net"),
    (r"\bbasic\b", "basic", r"\bdiluted\b", "diluted"),
    (r"\bcontrolling\b", "controlling", r"\bnon\s*controlling\b", "non-controlling"),
    (r"\bperforming\b", "performing", r"\bnon\s*performing\b", "non-performing"),
    (r"\btangible\b", "tangible", r"\bintangible\b", "intangible"),
    (r"\binterest\s+income\b|\bfinance\s+income\b", "finance income", r"\binterest\s+expense\b|\bfinance\s+costs?\b", "finance cost"),
    (r"\bfee\s+income\b", "fee income", r"\bfee\s+expenses?\b", "fee expense"),
    (r"\bopening\b|\bbeginning\b|\bstart\b", "beginning/opening", r"\bclosing\b|\bend\b", "end/closing"),
    (r"\bpaid\b", "paid", r"\breceived\b", "received"),
    (r"\btaxable\b", "taxable", r"\bexempt\b", "tax-exempt"),
    (r"\bcurrent\b", "current", r"\btotal\b", "total"),
    (r"\bnon\s*current\b", "non-current", r"\btotal\b", "total"),
    (r"\bauditors?\b|\baudit\b", "auditor", r"\bdirectors?\b", "director"),
    (r"\bchief\s+executive\b|\bmanaging\s+director\b|\bceo\b", "executive", r"\bdirectors?\b", "director"),
    (r"\binflows?\b", "inflow", r"\boutflows?\b", "outflow"),
]

GENERIC_LEGAL_SUFFIXES = [
    (re.compile(r"\b(public\s+limited\s+company|plc)\b", re.I), "plc"),
    (re.compile(r"\b(limited|ltd)\b", re.I), "ltd"),
    (re.compile(r"\b(private\s+limited|pvt\s+ltd|pvt)\b", re.I), "pvt ltd"),
    (re.compile(r"\b(incorporated|inc)\b", re.I), "inc"),
    (re.compile(r"\b(corporation|corp)\b", re.I), "corp"),
    (re.compile(r"\b(sdn\.?\s*bhd\.?)\b", re.I), "sdn bhd"),
    (re.compile(r"\b(llc|gmbh|sa|bv)\b", re.I), "llc"),
    (re.compile(r"\b(co\.|company)\b", re.I), "co"),
]

GENERIC_LEGAL_FORM_RE = re.compile(
    r"\b(ltd|limited|plc|pvt|private|inc|corp|corporation|llc|gmbh|co|company|sdn\.?\s*bhd\.?)\b",
    re.I
)

GENERIC_SEGMENT_TERMS = re.compile(
    r"\b(ltd|limited|plc|pvt|private|inc|corp|corporation|llc|gmbh|co|company|companies|"
    r"subsidiary|subsidiaries|branch|branches|brokerage|securities|capital|asset\s+management|"
    r"exchange\s+house|money\s+transfer|ventures|holdings|enterprises|pharma|textiles|power|energy|"
    r"conventional|islamic|takaful|offshore|onshore|retail|corporate|treasury|sme|"
    r"secured|unsecured|current\s+portion|total)\b",
    re.I
)

SEGMENT_DELIMITER_RE = re.compile(r"\s+[-–—]\s+|:\s*")

def normalize_entity_qualifier(text: str) -> str:
    """Normalizes an entity or segment qualifier for comparison across reporting years."""
    s = text.lower().strip()
    s = re.sub(r"[\.,\(\)]", " ", s)
    for pat, rep in GENERIC_LEGAL_SUFFIXES:
        s = pat.sub(rep, s)
    s = re.sub(r"\b(taka|tk|bdt|segment|segments|banking|branch|unit|units|operation|operations|division|divisions)\b", " ", s)
    return re.sub(r"\s+", " ", s).strip()

def extract_entity_or_segment_qualifier(field: str) -> Optional[str]:
    """Generic entity or segment qualifier of a line-item title (any company)."""
    f_clean = re.sub(r"\[.*?\]", "", field).strip()
    # 1. Parenthetical qualifiers, e.g. 'Other Assets (Subsidiary Name Ltd)'
    for p in re.findall(r"\(([^)]+)\)", f_clean):
        if GENERIC_SEGMENT_TERMS.search(p):
            return normalize_entity_qualifier(p)

    # 2. Delimited qualifiers, e.g. 'Note 43 - Other Assets - Subsidiary Ltd'
    parts = SEGMENT_DELIMITER_RE.split(f_clean)
    if len(parts) > 1:
        qualifier = parts[-1].strip()
        words = qualifier.split()
        is_entity = bool(GENERIC_SEGMENT_TERMS.search(qualifier)) or (len(words) >= 2 and all(w[0].isupper() for w in words if w))
        if is_entity:
            return normalize_entity_qualifier(qualifier)

    # 3. Standalone corporate legal entity name (e.g., in a subsidiary breakdown list)
    if GENERIC_LEGAL_FORM_RE.search(f_clean):
        if not re.search(r"\b(investment|investments|shares|advances|loans)\s+in\b", f_clean, re.I):
            return normalize_entity_qualifier(f_clean)

    return None

_TOTAL_WORDS = {"total", "subtotal"}


def _without_total_words(name: str) -> Tuple[str, ...]:
    """The words of a lower-case name without "total" / "subtotal" / "sub-total", sorted (word order does not matter)."""
    words = re.findall(r"\b\w+\b", name.replace("sub-total", "subtotal"))
    return tuple(sorted(w for w in words if w not in _TOTAL_WORDS))


def _same_line_subtotal_vs_total(n1: str, n2: str) -> bool:
    """"Cash subtotal" vs "Cash total": one report prints the sum row of a block as "Subtotal", another as "Total".
    Both names carry a total word and the rest is identical, so it is the same line, not a subtotal against its own
    component. A bare name against its subtotal ("Cash" vs "Cash subtotal") is still a contrast."""
    def has_total_word(n: str) -> bool:
        w = set(re.findall(r"\b\w+\b", n.replace("sub-total", "subtotal")))
        return bool(w & _TOTAL_WORDS)
    rest1 = _without_total_words(n1)
    return has_total_word(n1) and has_total_word(n2) and bool(rest1) and rest1 == _without_total_words(n2)


def has_aggregation_level_contrast(f1: str, f2: str) -> Tuple[bool, str]:
    """Contrast between aggregation levels, subtotals and subsidiary/segment breakdowns."""
    is_bracket1 = ("[" in f1 and "]" in f1)
    is_bracket2 = ("[" in f2 and "]" in f2)
    if is_bracket1 != is_bracket2:
        return True, "OCR-derived subtotal bracket contrast: '[...]' vs component"

    n1 = f1.lower().replace("[", " ").replace("]", " ")
    n2 = f2.lower().replace("[", " ").replace("]", " ")

    words1 = set(re.findall(r"\b\w+\b", n1))
    words2 = set(re.findall(r"\b\w+\b", n2))

    has_subtot1 = "subtotal" in words1 or "sub-total" in n1
    has_subtot2 = "subtotal" in words2 or "sub-total" in n2
    if has_subtot1 != has_subtot2 and not _same_line_subtotal_vs_total(n1, n2):
        return True, "Aggregation level contrast: subtotal vs component"

    has_tot1 = "total" in words1 or has_subtot1
    has_tot2 = "total" in words2 or has_subtot2

    is_other_component1 = ("other" in words1 and not has_tot1)
    is_other_component2 = ("other" in words2 and not has_tot2)

    if (is_other_component1 and has_tot2) or (is_other_component2 and has_tot1):
        return True, "Aggregation level contrast: 'other' residual component vs 'total' aggregate"

    sub_terms = {"subsidiary", "subsidiaries"}
    has_sub1 = any(s in words1 for s in sub_terms)
    has_sub2 = any(s in words2 for s in sub_terms)
    if has_sub1 != has_sub2:
        return True, "Entity scope contrast: subsidiary component vs parent/group"

    qual1 = extract_entity_or_segment_qualifier(f1)
    qual2 = extract_entity_or_segment_qualifier(f2)

    if qual1 or qual2:
        if qual1 and qual2 and qual1 != qual2:
            return True, f"Distinct entity or segment breakdown: '{qual1}' vs '{qual2}'"
        elif bool(qual1) != bool(qual2):
            q_found = qual1 or qual2
            return True, f"Entity/segment breakdown vs unsegmented account: '{q_found}'"

    return False, ""

_NEGATED_LABELS = {
    "non-current": "a", "non-operating": "b", "unsecured": "b", "indirect": "b", "non-controlling": "b",
    "non-performing": "b", "intangible": "b", "outside-bangladesh": "b",
}
_EXECUTIVE_RE = re.compile(r"\b(managing\s+director|chief\s+executive|deputy\s+managing\s+director)\b")

def has_unmergable_keyword_contrast(f1: str, f2: str) -> Tuple[bool, str]:
    """Detects mutually exclusive accounting classifications, contrasting modifiers, and aggregation levels."""
    is_agg, agg_reason = has_aggregation_level_contrast(f1, f2)
    if is_agg:
        return True, agg_reason

    n1 = " " + normalize_text(f1) + " "
    n2 = " " + normalize_text(f2) + " "

    for pat_a, lbl_a, pat_b, lbl_b in CONTRAST_SPECS:
        match_a1 = bool(re.search(pat_a, n1))
        match_b1 = bool(re.search(pat_b, n1))
        match_a2 = bool(re.search(pat_a, n2))
        match_b2 = bool(re.search(pat_b, n2))

        # One pattern is a negation/superset of the other ("non-current" contains "current"):
        # when the more specific side matches, the generic side does not count.
        for lbl, side in _NEGATED_LABELS.items():
            if lbl in lbl_a and side == "a":
                if match_a1:
                    match_b1 = False
                if match_a2:
                    match_b2 = False
            if lbl in lbl_b and side == "b":
                if match_b1:
                    match_a1 = False
                if match_b2:
                    match_a2 = False
        if "director" in lbl_b and _EXECUTIVE_RE.search(n1):
            match_b1 = False
        if "director" in lbl_b and _EXECUTIVE_RE.search(n2):
            match_b2 = False

        if (match_a1 and not match_b1 and match_b2 and not match_a2) or (match_b1 and not match_a1 and match_a2 and not match_b2):
            return True, f"Mutually exclusive IFRS classification: '{lbl_a}' vs '{lbl_b}'"

    return False, ""

def extract_core_concept_head(field: str) -> str:
    """The underlying account of a line-item title, with boilerplate wrappers removed:
    trailing legal-entity names, cash-flow movement prefixes ("Increase/(decrease) in ..."),
    "Other assets - X" category prefixes and a leading "Consolidated"/"Standalone"."""
    s = re.sub(r"\[|\]", "", str(field)).strip()

    # Trailing legal entity names, e.g. ' - Example Holdings Limited'
    s = re.sub(
        r"\s*[-–—]\s*(the\s+)?([a-z0-9\s]+?)\s*(limited|ltd|plc|pvt|corp|corporation|bank|inc|holdings|enterprises|exchange\s+house|securities)\b.*$",
        "", s, flags=re.I
    )

    movement_prefixes = [
        r"^\s*(consolidated\s+)?(increase\s+(or|and|\/)\s*decrease\s+(in|of)?)\s*",
        r"^\s*(consolidated\s+)?(increase\s*\/\s*\(decrease\)\s*(in|of)?)\s*",
        r"^\s*(consolidated\s+)?(net\s+(increase|decrease)\s+(in|of)?)\s*",
        r"^\s*(consolidated\s+)?(net\s+cash\s+(flows?\s+)?(from|used\s+in|generated\s+from|provided\s+by)?)\s*",
        r"^\s*(consolidated\s+)?(cash\s+flows?\s+(from|used\s+in|generated\s+from|provided\s+by)?)\s*",
        r"^\s*(consolidated\s+)?(proceeds\s+from|payments?\s+(for|to)|receipts?\s+(from|for)|adjustments?\s+(for|to))\s*",
        r"^\s*(consolidated\s+)?(net\s+change\s+in|changes?\s+in)\s*",
    ]
    for pat in movement_prefixes:
        s = re.sub(pat, "", s, flags=re.I).strip()

    cat_match = re.match(r"^(other\s+assets|other\s+liabilities)\s*[-–—:]\s*(.+)$", s, flags=re.I)
    if cat_match:
        s = cat_match.group(2).strip()

    s = re.sub(r"^\s*(consolidated|standalone)\s+", "", s, flags=re.I).strip()
    return normalize_text(s)

def are_candidate_pair(f1: str, f2: str) -> Tuple[bool, str]:
    """True when two titles are lexically plausible synonyms (compared on their core concept heads)."""
    if f1 == f2:
        return True, "exact"
    if normalize_text(f1) == normalize_text(f2):
        return True, "norm_exact"
    if apply_synonyms(f1) == apply_synonyms(f2):
        return True, "synonym_exact"

    h1 = extract_core_concept_head(f1)
    h2 = extract_core_concept_head(f2)
    if h1 == h2 and len(h1) > 2:
        return True, "core_head_exact"

    syn_h1 = apply_synonyms(h1)
    syn_h2 = apply_synonyms(h2)
    if syn_h1 == syn_h2 and len(syn_h1) > 2:
        return True, "core_head_synonym"

    tok1 = get_tokens(syn_h1)
    tok2 = get_tokens(syn_h2)
    union_len = len(tok1.union(tok2))
    if union_len > 0:
        jaccard = len(tok1.intersection(tok2)) / union_len
        if jaccard >= 0.65:
            return True, f"core_jaccard_{jaccard:.2f}"

    ratio = SequenceMatcher(None, syn_h1, syn_h2).ratio()
    if ratio >= 0.78:
        return True, f"core_seq_sim_{ratio:.2f}"

    if (syn_h1 in syn_h2 or syn_h2 in syn_h1) and min(len(tok1), len(tok2)) >= 2:
        diff = tok1.symmetric_difference(tok2)
        if diff.issubset({"off", "from", "for", "to", "in", "and", "the", "period", "during", "at", "of", "as"}):
            return True, "modifier_diff"

    if syn_h1 in syn_h2 or syn_h2 in syn_h1:
        shorter_toks = tok1 if len(tok1) <= len(tok2) else tok2
        longer_toks = tok2 if len(tok1) <= len(tok2) else tok1
        if len(shorter_toks) >= 2 and (len(shorter_toks) / len(longer_toks)) >= 0.45:
            is_cont, _ = has_unmergable_keyword_contrast(f1, f2)
            if not is_cont:
                return True, "containment_candidate"

    if "number of shares" in syn_h1 and "number of shares" in syn_h2:
        return True, "share_count_candidate"

    return False, ""

# ---------------------------------------------------------------------------
# Gemini judge for ambiguous cross-report links
# ---------------------------------------------------------------------------
_gemini_client: Optional[genai.Client] = None
_rate_limiter = rate_limiters["merge"]


def get_gemini_client() -> genai.Client:
    global _gemini_client
    if _gemini_client is None:
        key = gemini_api_key("merge")
        if not key:
            raise RuntimeError("MERGE_API_KEY is not configured in .env or environment")
        _gemini_client = genai.Client(api_key=key)
        logger.info(f"Initialized Gemini Client for multi-year series linking using {DEFAULT_MODEL}")
    return _gemini_client


# Account type codes used in the compact link output
ACCOUNT_TYPE_CODES = {
    "A": "ASSET", "L": "LIABILITY", "E": "EQUITY", "R": "REVENUE", "X": "EXPENSE",
    "C": "CASH_FLOW_ITEM", "N": "OPERATIONAL_COUNT", "O": "OTHER",
}

LINK_SYSTEM_INSTRUCTION = """You are an expert financial auditor reconciling annual reports of the same company across years.
Each question gives one line item (A) from an earlier annual report and up to five candidate line items from a later annual report (usually the next one, sometimes a few years later) of the same statement and reporting scope.
Decide which candidate, if any, is THE SAME line item as A (the same account, possibly re-worded or renamed), so the two form one continuous time-series.

EVIDENCE:
- Every report prints the previous year as a comparative column. The evidence token tells you how the candidate's figure compares with A's figure for a year both reports print:
  "=2020:1,234,567" identical value for 2020 (strong evidence of the same item, e.g. a rename like 'Accounts Payable' -> 'Creditors and Other Payables');
  "≠2020:+3.2%" the values differ by that percentage (a restatement at most; a large difference means a different item);
  "∅" no overlapping year (e.g. the two reports are several years apart): judge by the names and the statement alone, and be strict:
  answer only when the candidate is clearly the same account re-worded (for example "Total Net Profit After Tax Attributable" and "Net profit after tax attributable to total").
- Identical values are NOT enough when the two lines are different accounts that are equal by an accounting identity or by coincidence
  (e.g. 'Total Assets' vs 'Total Equity and Liabilities', 'Cash' vs 'Total Cash', 'Advance Tax Paid' vs 'Minimum Tax').
- Entity words: when two names differ ONLY by an entity or ownership word (company / companies, parent, group, subsidiary / subsidiaries, associate)
  and the evidence token shows an identical value ("=") for a year both reports print, the word is a re-wording, not a different account: answer SAME
  (e.g. 'Investment in Associate Companies' -> 'Investment in Associate and Subsidiary Companies' with "=2019:256,342,357";
  'Equity attributable to owners of the company' -> 'Equity attributable to owners of the parent'). With "∅" answer SAME only when the rest of
  the name is the same standard account.
- Standard totals: statement-level totals have standard names, and a name that differs only by filler words (net, total, the, in, of, and,
  "increase and decrease" vs "increase / (decrease)", "cash" vs "cash and cash equivalents") is the same line even with "∅", when the statement and
  section agree and the candidate is the only such total (e.g. 'Net Cash Increase / (Decrease)' -> 'Net Increase and Decrease in Cash and Cash Equivalents').
  A subtotal is never the same line as one of its own components.
- Never link: current vs non-current; operating vs investing vs financing; assets vs liabilities vs equity; income vs expense;
  gross vs net; basic vs diluted; opening vs closing balances; a component vs a total/subtotal; differently NAMED subsidiaries or segments
  (for example "X Ltd" vs "Y Ltd"; an entity word alone is covered by the rule above).

INPUT: {"f":[label_0, label_1, ...], "q":[[question_id, "STATEMENT", a_index, a_report, a_section, [[candidate_index, candidate_report, candidate_section, evidence], ...]], ...]}
Labels are referenced by their index in "f". Sections: CA/NCA current/non-current assets, CL/NCL current/non-current liabilities, EQ equity,
REV revenue, COS cost of sales, OPEX expenses, FIN finance, TAX tax, OCI other comprehensive income, CF_OP/CF_INV/CF_FIN/CF_CASH cash-flow sections, PER_SHARE, OTHER.

OUTPUT (JSON following the response schema; keep it short):
{"t":"<account type codes>","d":[{"q":question_id,"c":choice,"v":verdict},...]}
- "t": one character per entry of "f", in order: A=ASSET, L=LIABILITY, E=EQUITY, R=REVENUE, X=EXPENSE, C=CASH_FLOW_ITEM, N=OPERATIONAL_COUNT, O=OTHER.
- "d": one row for EVERY question, in the order given. "c" is the 0-based position of the candidate that is most likely the same line in that question's
  candidate list, or -1 when none is. "v" is how sure you are: "SAME" only when the two items are certainly the same account;
  "UNSURE" when it is plausible but not certain; "DIFFERENT" when no candidate is the same account (then use "c": -1)."""

LINK_RESPONSE_SCHEMA: Dict[str, Any] = {
    "type": "OBJECT",
    "properties": {
        "t": {"type": "STRING"},
        "d": {
            "type": "ARRAY",
            "items": {
                "type": "OBJECT",
                "properties": {
                    "q": {"type": "INTEGER"},
                    "c": {"type": "INTEGER"},
                    "v": {"type": "STRING", "enum": ["SAME", "UNSURE", "DIFFERENT"]},
                },
                "required": ["q", "c", "v"],
                "propertyOrdering": ["q", "c", "v"],
            },
        },
    },
    "required": ["t", "d"],
    "propertyOrdering": ["t", "d"],
}


VERDICT_CONFIDENCE = {"SAME": 0.95, "UNSURE": 0.5, "DIFFERENT": 0.0}


def _question_sort_key(q: Dict[str, Any]) -> Tuple:
    """A content key, so the same questions are always asked in the same order (the prompt is a pure function of the data)."""
    head = q["head"]
    return (q.get("partition") or "", q.get("head_key") or head["label"], head["report"],
            tuple(c.get("key") or c["label"] for c in q["candidates"]))


def split_link_batches(questions: List[Dict[str, Any]], batch_size: int) -> List[List[Dict[str, Any]]]:
    """Questions sorted by content and cut into batches on partition boundaries: whole partitions are packed
    together, so a change in one partition cannot move questions of another into a different batch. A partition
    larger than a batch is split in content order."""
    ordered = sorted(questions, key=_question_sort_key)
    groups: List[List[Dict[str, Any]]] = []
    for q in ordered:
        if groups and (groups[-1][0].get("partition") or "") == (q.get("partition") or ""):
            groups[-1].append(q)
        else:
            groups.append([q])
    batches: List[List[Dict[str, Any]]] = []
    current: List[Dict[str, Any]] = []
    for g in groups:
        if len(g) > batch_size:
            if current:
                batches.append(current)
                current = []
            batches.extend(g[i:i + batch_size] for i in range(0, len(g), batch_size))
        elif len(current) + len(g) > batch_size:
            batches.append(current)
            current = list(g)
        else:
            current.extend(g)
    if current:
        batches.append(current)
    return batches


def build_link_batch_input(questions: List[Dict[str, Any]]) -> Tuple[Dict[str, Any], List[str]]:
    """Compact batch input: each distinct label once; questions reference labels by index.
    Question ids in the prompt are the positions 0..n-1 inside the batch (never a global counter)."""
    labels: List[str] = []
    index: Dict[str, int] = {}

    def idx(name: str) -> int:
        if name not in index:
            index[name] = len(labels)
            labels.append(name)
        return index[name]

    rows = []
    for pos, q in enumerate(questions):
        head = q["head"]
        head_idx = idx(head["label"])
        cands = [[idx(c["label"]), c["report"], c["section"], c["evidence"]] for c in q["candidates"]]
        rows.append([pos, q["statement_type"], head_idx, head["report"], head["section"], cands])
    return {"f": labels, "q": rows}, labels


def decode_link_batch_output(parsed: Any, labels: List[str]) -> Tuple[Dict[int, Dict[str, Any]], Dict[str, str]]:
    """{question id in the prompt: {"choice": int (-1 = none), "verdict", "confidence"}}, {label: account type name or ""}."""
    if not isinstance(parsed, dict):
        return {}, {}
    type_codes = str(parsed.get("t") or "")
    types_by_label = {
        name: ACCOUNT_TYPE_CODES.get(type_codes[i].upper(), "") if i < len(type_codes) else ""
        for i, name in enumerate(labels)
    }
    out: Dict[int, Dict[str, Any]] = {}
    for row in parsed.get("d") or []:
        if isinstance(row, dict):
            row = [row.get("q"), row.get("c"), row.get("v")]
        if not isinstance(row, (list, tuple)) or len(row) < 3:
            continue
        try:
            qid, choice = int(row[0]), int(row[1])
        except (TypeError, ValueError):
            continue
        verdict = str(row[2] or "").strip().upper()
        if verdict not in VERDICT_CONFIDENCE:
            continue
        if choice < 0:
            verdict = "DIFFERENT"
        out[qid] = {"choice": choice, "verdict": verdict, "confidence": VERDICT_CONFIDENCE[verdict]}
    return out, types_by_label


LINK_BATCH_SIZE = 50        # questions per Gemini call


def judge_link_questions(
    questions: List[Dict[str, Any]],
    batch_size: int = LINK_BATCH_SIZE,
    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
    should_cancel: Optional[Callable[[], bool]] = None,
    model_name: str = DEFAULT_MODEL,
) -> Tuple[Dict[int, Dict[str, Any]], Dict[str, Any]]:
    """Asks Gemini which candidate (if any) continues each question's line item.

    questions: [{"qid", "statement_type", "head": {"label","report","section"},
                 "candidates": [{"label","report","section","evidence"}]}]
    Returns ({qid: {"choice", "confidence", "type_head", "type_choice"}}, usage).
    Raises MergeCancelled before each Gemini call once should_cancel() returns True.

    progress_callback gets {"type": "progress", "stage": "gemini_call_started" | "gemini_retry" | "gemini_call_done", ...}
    around every call (call_in_round / calls_in_round, call_duration_sec on "done"); evaluated_pairs, matched_pairs and
    conflicts_count count this call set only (the merge orchestrator turns them into whole-merge numbers)."""
    usage = {"prompt_tokens": 0, "candidates_tokens": 0, "total_tokens": 0, "batches_count": 0, "batch_details": [],
             "missing_answers": 0}
    if not questions:
        return {}, usage
    client = get_gemini_client()
    answers: Dict[int, Dict[str, Any]] = {}
    done = 0
    batches = split_link_batches(questions, batch_size)

    def emit(stage: str, b_idx: int, chunk: List[Dict[str, Any]], subtitle: str, **extra) -> None:
        if progress_callback:
            progress_callback({
                "type": "progress",
                "stage": stage,
                "percent": 10 + int(done / len(questions) * 80),
                "total_pairs": len(questions),
                "evaluated_pairs": done,
                "matched_pairs": sum(1 for a in answers.values() if a["choice"] >= 0),
                "conflicts_count": sum(1 for a in answers.values() if a["choice"] < 0),
                "current_statement": chunk[-1]["statement_type"],
                "field_a": chunk[-1]["head"]["label"],
                "field_b": "",
                "status": "EVALUATED" if stage == "gemini_call_done" else "EVALUATING",
                "confidence": 0.0,
                "call_in_round": b_idx + 1,
                "calls_in_round": len(batches),
                "questions_in_call": len(chunk),
                "subtitle": subtitle,
                **extra,
            })

    for b_idx, chunk in enumerate(batches):
        emit("gemini_call_started", b_idx, chunk, f"Asking Fintra AI about {len(chunk)} ambiguous links")
        call_started = time.time()
        batch_input, labels = build_link_batch_input(chunk)
        prompt = ("Decide which candidate continues each line item.\n\n"
                  f"{json.dumps(batch_input, separators=(',', ':'), ensure_ascii=False)}")
        response = None
        for attempt in range(4):
            _check_cancelled(should_cancel)
            try:
                _rate_limiter.wait_sync()
                response = generate_json(client, model_name, prompt, LINK_SYSTEM_INSTRUCTION,
                                         temperature=0.0, response_schema=LINK_RESPONSE_SCHEMA)
                break
            except Exception as e:
                err = str(e)
                if attempt < 3 and any(k in err for k in ("503", "429", "UNAVAILABLE", "RESOURCE_EXHAUSTED")):
                    sleep_time = (2 ** attempt) * 2
                    logger.warning("Gemini temporary error on link batch %d (%s); retrying in %ss", b_idx + 1, err, sleep_time)
                    emit("gemini_retry", b_idx, chunk, f"Fintra AI is busy, retrying (attempt {attempt + 2} of 4)",
                         retry_attempt=attempt + 2, retry_in_sec=sleep_time)
                    time.sleep(sleep_time)
                else:
                    raise

        tok = usage_tokens(response)
        usage["prompt_tokens"] += tok["prompt_tokens"]
        usage["candidates_tokens"] += tok["candidates_tokens"]
        usage["total_tokens"] += tok["total_tokens"]
        usage["batches_count"] += 1

        try:
            decoded, types_by_label = decode_link_batch_output(json.loads(response_text(response) or "{}"), labels)
        except ValueError as e:
            logger.error("Unparseable Gemini link answer: %s", e)
            decoded, types_by_label = {}, {}
        missing = 0
        for pos, q in enumerate(chunk):
            ans = decoded.get(pos)
            if ans is None:
                # an answer row the model left out is NOT a "no": it is reported as NO_ANSWER
                missing += 1
                answers[q["qid"]] = {"choice": -1, "verdict": "NO_ANSWER", "confidence": 0.0,
                                     "type_head": "", "type_choice": ""}
                continue
            choice = ans["choice"]
            chosen = q["candidates"][choice] if 0 <= choice < len(q["candidates"]) else None
            answers[q["qid"]] = {
                "choice": choice if chosen else -1,
                "verdict": ans["verdict"] if chosen else "DIFFERENT",
                "confidence": ans["confidence"] if chosen else 0.0,
                "type_head": types_by_label.get(q["head"]["label"], ""),
                "type_choice": types_by_label.get(chosen["label"], "") if chosen else "",
            }

        usage["missing_answers"] += missing
        usage["batch_details"].append({"batch_index": b_idx + 1, "questions_in_batch": len(chunk),
                                       "missing_answers": missing, **tok})
        if missing:
            logger.warning("Gemini left out %d of %d answers in link batch %d (reported as NO_ANSWER)",
                           missing, len(chunk), b_idx + 1)
        done += len(chunk)
        emit("gemini_call_done", b_idx, chunk, f"Fintra AI checked {done}/{len(questions)} ambiguous links",
             call_duration_sec=round(time.time() - call_started, 2))
    return answers, usage
