"""Runs the multi-year merge for one company: relabel -> nodes -> series links -> saved state -> merged rows.

The result has the shape the merge endpoints and /results/merged already use (rows, merge_report, conflicts,
csv_text ...), plus additive series fields. See services/series_linker.py for the linking rules."""

import copy
import logging
import time
from typing import Any, Callable, Dict, List, Optional

from .. import database
from ..config import DEFAULT_MODEL
from . import merge_service, series_linker
from .merge_service import MergeCancelled, _check_cancelled

logger = logging.getLogger("ocr.series_merge")


def plan_gemini_calls(nodes: List[Dict[str, Any]], overrides: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Dry run of the linker (no Gemini, nothing saved): the rounds that will ask Gemini, as
    [{"round": report label or "JOIN", "questions": n, "calls": n}]. An estimate: a real Gemini "same" answer
    changes which lines are open in the next report, so the real rounds can differ a little (the live progress
    corrects the estimate round by round). Returns [] when the dry run fails."""
    rounds: List[Dict[str, Any]] = []
    current: Dict[str, Any] = {}

    def on_questions(label: str, n: int) -> None:
        current["round"] = label

    def judge(questions: List[Dict[str, Any]]) -> Dict[int, Dict[str, Any]]:
        rounds.append({"round": current.get("round", ""), "questions": len(questions),
                       "calls": len(merge_service.split_link_batches(questions, merge_service.LINK_BATCH_SIZE))})
        return {}

    try:
        series_linker.link_company(copy.deepcopy(nodes), overrides, judge=judge, on_questions=on_questions)
    except Exception:
        logger.exception("Could not estimate the Gemini calls of the merge")
        return []
    return rounds


# Who decided a link: the system (name, value and gap rules), a user override, or Fintra AI.
_DECIDER = {"user": "user", "auto": "system", "gemini": "fintra_ai"}


class MergeStats:
    """The "how far is the merge" numbers sent with every progress event and in the final summary (`merge_stats`).

    reports: how many reports are linked (a report is linked to the ones before it). lines: the printed line items
    (one per report and statement line) of those reports, `done` once their report is linked, `remaining` the rest.
    links: the pairs of lines found to be the same line, by who decided (system / user / fintra_ai). fintra_ai: the questions
    sent to Fintra AI. reviews: items made for a person to look at (while the merge runs only those the linking makes;
    the final summary has them all)."""

    def __init__(self, nodes: List[Dict[str, Any]]):
        reports = sorted({n["report"] for n in nodes}, key=lambda r: (_end_year(r), r))
        self.reports_total = len(reports)
        self.lines_by_report = [sum(1 for n in nodes if n["report"] == r) for r in reports]
        self.lines_total = len(nodes)
        self.reports_done = 0
        self.links: List[Dict[str, Any]] = []
        self.review: List[Dict[str, Any]] = []
        self.auto_resolved = 0
        self.final = False

    def step(self, done: int, total: int, links: List[Dict[str, Any]], review: List[Dict[str, Any]]) -> None:
        self.reports_done, self.links, self.review = done, links, review

    def finish(self, links: List[Dict[str, Any]], review: List[Dict[str, Any]]) -> None:
        self.reports_done, self.links, self.review, self.final = self.reports_total, links, review, True
        self.auto_resolved = sum(1 for i in review if i.get("status") == "resolved")

    def snapshot(self, ai: Optional[Dict[str, int]] = None) -> Dict[str, Any]:
        lines_done = sum(self.lines_by_report[:self.reports_done])
        by = {"system": 0, "user": 0, "fintra_ai": 0}
        for l in list(self.links):
            by[_DECIDER.get(l.get("source"), "system")] += 1
        levels = {"high": 0, "medium": 0, "low": 0}
        for i in list(self.review):
            levels[series_linker.review_attention(i["kind"], i["payload"]).lower()] += 1
        ai = ai or {}
        asked = ai.get("questions_total", 0)
        answered = asked if self.final else ai.get("questions_answered", 0)      # a finished merge has had every question answered
        return {
            "reports": {"total": self.reports_total, "done": self.reports_done, "remaining": self.reports_total - self.reports_done},
            "lines": {"total": self.lines_total, "done": lines_done, "remaining": self.lines_total - lines_done},
            "links": {"total": sum(by.values()), "by_system": by["system"], "by_user": by["user"], "by_fintra_ai": by["fintra_ai"]},
            "fintra_ai": {"questions_total": asked, "questions_answered": answered, "questions_remaining": max(asked - answered, 0),
                          "linked": by["fintra_ai"], "not_linked": max(answered - by["fintra_ai"], 0),
                          "calls_total": ai.get("calls_total", 0), "calls_done": ai.get("calls_done", 0)},
            "reviews": {"created": len(self.review), "auto_resolved": self.auto_resolved, "high": levels["high"],
                        "medium": levels["medium"], "low": levels["low"], "complete": self.final},
        }


def _end_year(report: str) -> int:
    from . import series_keys as sk
    return sk.report_end_year(report)


class GeminiProgress:
    """Turns the per-call events of merge_service.judge_link_questions into whole-merge progress.

    Gemini is asked once per report transition (and once for the final join pass), so the number of calls is
    only exact when the merge is done. It starts as the dry-run plan and is corrected as rounds are reached: a
    round that has started has its real call count, rounds not reached yet keep their planned count."""

    def __init__(self, emit: Callable[[Dict[str, Any]], None], plan: List[Dict[str, Any]], stats: Optional[MergeStats] = None):
        self.emit = emit
        self.stats = stats
        self.plan = {r["round"]: r for r in plan}
        self.started = time.time()
        self.seen: List[str] = []
        self.calls_before = self.questions_before = self.matched_before = self.unmatched_before = 0
        self.round_calls = self.round_questions = 0
        self.round_matched = self.round_unmatched = self.round_calls_done = self.round_questions_done = 0
        self.call_seconds: List[float] = []
        self.percent = 10

    # -- numbers
    def _planned_ahead(self, key: str) -> int:
        return sum(r[key] for name, r in self.plan.items() if name not in self.seen)

    @property
    def calls_total(self) -> int:
        return self.calls_before + self.round_calls + self._planned_ahead("calls")

    @property
    def questions_total(self) -> int:
        return self.questions_before + self.round_questions + self._planned_ahead("questions")

    @property
    def calls_done(self) -> int:
        return self.calls_before + self.round_calls_done

    def fields(self, exact: bool = False) -> Dict[str, Any]:
        """The fields every init / progress / finalize event carries."""
        remaining = max(self.calls_total - self.calls_done, 0)
        eta = round(sum(self.call_seconds) / len(self.call_seconds) * remaining, 1) if self.call_seconds else None
        ai = {"questions_total": self.questions_total, "questions_answered": self.questions_before + self.round_questions_done,
              "linked": self.matched_before + self.round_matched, "not_linked": self.unmatched_before + self.round_unmatched,
              "calls_total": self.calls_total, "calls_done": self.calls_done}
        return {
            **({"merge_stats": self.stats.snapshot(ai)} if self.stats is not None else {}),
            "gemini_calls_total": self.calls_total, "gemini_calls_done": self.calls_done, "estimate_exact": exact,
            "questions_total": self.questions_total,
            "round": self.seen[-1] if self.seen else None, "round_index": len(self.seen),
            "rounds_total": max(len(self.plan), len(self.seen)),
            "elapsed_sec": round(time.time() - self.started, 1), "eta_sec": eta,
        }

    # -- hooks
    def round_started(self, label: str, n_questions: int) -> None:
        """link_company's on_questions: the next judge call belongs to this round."""
        self.calls_before += self.round_calls
        self.questions_before += self.round_questions
        self.matched_before += self.round_matched
        self.unmatched_before += self.round_unmatched
        self.seen.append(label)
        self.round_questions = n_questions
        self.round_calls = self.plan.get(label, {}).get("calls", 0)
        self.round_calls_done = self.round_questions_done = self.round_matched = self.round_unmatched = 0

    def step(self, done: int, total: int, links: List[Dict[str, Any]], review: List[Dict[str, Any]]) -> None:
        """link_company's on_step: a report is linked. One event per report, so a merge that asks Fintra AI nothing still moves."""
        if self.stats is not None:
            self.stats.step(done, total, links, review)
        self.percent = max(self.percent, min(10 + int(80 * done / max(total, 1)), 90))
        f = self.fields()
        self.emit({"type": "progress", "stage": "report_linked", "percent": self.percent, "total_pairs": f["questions_total"],
                   "evaluated_pairs": self.questions_before + self.round_questions_done,
                   "matched_pairs": self.matched_before + self.round_matched,
                   "conflicts_count": self.unmatched_before + self.round_unmatched, **f,
                   "subtitle": f"Linked {done} of {total} reports"})

    def on_event(self, e: Dict[str, Any]) -> None:
        """merge_service's progress_callback: the round's own counters become whole-merge ones."""
        finished = e["stage"] == "gemini_call_done"
        self.round_calls = e["calls_in_round"]
        self.round_calls_done = e["call_in_round"] if finished else e["call_in_round"] - 1
        self.round_questions_done = e["evaluated_pairs"]
        self.round_matched, self.round_unmatched = e["matched_pairs"], e["conflicts_count"]
        if finished:
            self.call_seconds.append(e.get("call_duration_sec", 0.0))
        f = self.fields()
        # never backwards, even when a round turns out to need more calls than planned
        self.percent = max(self.percent, min(10 + int(80 * f["gemini_calls_done"] / max(f["gemini_calls_total"], 1)), 90))
        subtitle = e["subtitle"]
        if e["stage"] == "gemini_call_started":
            subtitle = f"Asking Fintra AI (call {f['gemini_calls_done'] + 1} of about {f['gemini_calls_total']})"
        elif finished:
            subtitle = f"Fintra AI call {f['gemini_calls_done']} of about {f['gemini_calls_total']} done"
        self.emit({**e, **f, "percent": self.percent, "subtitle": subtitle,
                   "total_pairs": f["questions_total"],
                   "evaluated_pairs": self.questions_before + self.round_questions_done,
                   "matched_pairs": self.matched_before + self.round_matched,
                   "conflicts_count": self.unmatched_before + self.round_unmatched})


def _default_judge(progress_callback, should_cancel, usage: Dict[str, Any]):
    def judge(questions: List[Dict[str, Any]]) -> Dict[int, Dict[str, Any]]:
        answers, u = merge_service.judge_link_questions(
            questions, progress_callback=progress_callback, should_cancel=should_cancel)
        for k in ("prompt_tokens", "candidates_tokens", "total_tokens", "batches_count", "missing_answers"):
            usage[k] += u.get(k, 0)
        usage["batch_details"].extend(u.get("batch_details", []))
        return answers
    return judge


def _orphan_override_items(overrides: List[Dict[str, Any]], nodes: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Overrides whose labels match nothing in the current data (e.g. after a re-extraction renamed a line)."""
    from . import series_keys as sk
    labels = {(n["partition"], n["label"]) for n in nodes}
    out = []
    for o in overrides:
        pkey = series_linker.partition_key(o["statement_type"], o["scope"], o.get("entity_key") or "", o.get("segment_key") or "")
        missing = [k for k in ("a_label_key", "b_label_key") if o.get(k) and (pkey, sk.label_key(o[k])) not in labels]
        if missing:
            out.append({"kind": "ORPHAN_OVERRIDE", "dedupe_key": str(o["id"]),
                        "payload": {"override_id": o["id"], "override_kind": o["kind"], "missing": [o[k] for k in missing],
                                    "statement_type": o["statement_type"], "scope": o["scope"]}})
    return out


def _stale_link_items(previous: List[Dict[str, Any]], links: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Gemini links of the previous merge that the current data no longer supports."""
    now = {(l["partition"], l["from_node"], l["to_node"]) for l in links}
    out = []
    for p in previous:
        if p["source"] != "gemini" or (p["partition_key"], p["from_node"], p["to_node"]) in now:
            continue
        out.append({"kind": "STALE_LINK", "dedupe_key": f"{p['from_node']}>{p['to_node']}",
                    "payload": {"partition": p["partition_key"], "a_label_key": p["from_label"], "a_report": p["from_report"],
                                "b_label_key": p["to_label"], "b_report": p["to_report"], "rule": p["rule"],
                                "a_node": p["from_node"], "b_node": p["to_node"],
                                "confidence": p["confidence"]}})
    return out


# Review kinds raised by the extraction of a page (not by a merge): see database.replace_page_review_items.
EXTRACTION_FLAGS = ("FIELD_UNIDENTIFIED", "LABEL_EVIDENCE_FAILED")


def _node_flag_items(nodes: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    out = []
    for n in nodes:
        for flag in n["flags"]:
            if flag in EXTRACTION_FLAGS:       # reviewed right after the extraction of the page (database.replace_page_review_items)
                continue
            out.append({"kind": flag, "dedupe_key": f"{n['partition']}#{n['id']}",
                        "payload": {"partition": n["partition"], "report": n["report"], "field": n["fields"][0] if n["fields"] else n["label"],
                                    "label_key": n["label"], "category": n["category"],
                                    "kept": {str(y): {**{k: p.get(k) for k in ("page_number", "table_num", "table_row", "table_col")},
                                                      "value": n["values"][y][0] if y in n["values"] else None}
                                             for y, p in n["prov"].items() if y in n["alt_periods"]},
                                    "alt_periods": {str(y): v for y, v in n["alt_periods"].items()}}})
    return out


def _first_page(node: Dict[str, Any]) -> Optional[int]:
    for y in sorted(node["prov"], reverse=True):
        pg = (node["prov"][y] or {}).get("page_number")
        if pg is not None:
            return pg
    return None


def _timeline_line(role: str, entry: Dict[str, Any], node_id: str, series: Dict[str, Any]) -> Dict[str, Any]:
    """One line of a timeline: the series' resolved values per year plus each report's own entry."""
    from . import series_keys as sk
    nodes = series["nodes"]
    resolved = series_linker.resolve_series_values(series)["values"]
    stating: Dict[int, str] = {}
    best: Dict[int, tuple] = {}
    for n in nodes:
        for y in n["values"]:
            k = (n["report_end"], n["report"])
            if y not in best or k > best[y]:
                best[y] = k
                stating[y] = n["report"]
    return {"role": role, "report": entry.get("report"), "field": entry.get("field"), "label_key": entry.get("label_key"),
            "node_id": node_id, "first_report": nodes[0]["report"], "last_report": nodes[-1]["report"],
            "_values": {y: float(v) for y, v in resolved.items()}, "_stating": stating,
            "members": [{"report": n["report"], "field": series_linker._display(n), "label_key": n["label"],
                         "node_id": n["id"], "page_number": _first_page(n),
                         "values": {str(y): float(v[0]) for y, v in sorted(n["values"].items())}} for n in nodes]}


def _attach_timelines(review: List[Dict[str, Any]], series: List[Dict[str, Any]], nodes: List[Dict[str, Any]]) -> None:
    """Adds payload["timeline"] (every year of every involved line, across all reports) to SPLIT_DETECTED and
    MERGE_DETECTED items, in place. node_ids is [the single node, *parts]; the single node is `from` for a split
    and `into` for a merge. Purely derived from the built series; nothing is changed or re-asked."""
    from . import series_keys as sk
    series_of = {(n["partition"], n["id"]): s for s in series for n in s["nodes"]}
    node_by = {(n["partition"], n["id"]): n for n in nodes}
    for item in review:
        if item["kind"] not in ("SPLIT_DETECTED", "MERGE_DETECTED"):
            continue
        p = item["payload"]
        ids = p.get("node_ids") or []
        if item["kind"] == "SPLIT_DETECTED":
            plan = [("from", p["from"], ids[0])] + [("into", e, i) for e, i in zip(p["into"], ids[1:])]
        else:
            plan = [("into", p["into"], ids[0])] + [("from", e, i) for e, i in zip(p["from"], ids[1:])]
        plan.sort(key=lambda t: t[0] != "from")
        lines = []
        for role, entry, nid in plan:
            s = series_of.get((p["partition"], nid))
            if s is None and (p["partition"], nid) in node_by:
                s = {"partition": p["partition"], "nodes": [node_by[(p["partition"], nid)]], "links": []}
            if s is None:
                continue
            lines.append((_timeline_line(role, entry, nid, s), s))
        if not lines:
            continue
        years = sorted({y for l, _ in lines for y in l["_values"]})
        reports = sorted({n["report"] for _, s in lines for n in s["nodes"]}, key=lambda r: (sk.report_end_year(r), r))
        out_lines = []
        for l, _ in lines:
            vals, stat = l.pop("_values"), l.pop("_stating")
            l["values"] = {str(y): vals.get(y) for y in years}
            l["stating_reports"] = {str(y): stat[y] for y in years if y in vals}
            out_lines.append(l)
        p["timeline"] = {"years": years, "reports": reports, "lines": out_lines}


def _comparison_line(side: str, node: Dict[str, Any]) -> Dict[str, Any]:
    """One side of a value conflict: the line's whole row (every year this report prints) with the printed cell of each."""
    return {"side": side, "report": node["report"], "field": series_linker._display(node), "label_key": node["label"],
            "node_id": node["id"], "section": node["section"], "unit": node["unit"],
            "values": {str(y): float(v[0]) for y, v in sorted(node["values"].items())},
            "cells": {str(y): {"page_number": pv.get("page_number"), "table_num": pv.get("table_num"),
                               "table_row": pv.get("table_row"), "table_col": pv.get("table_col")}
                      for y, pv in sorted(node["prov"].items()) if y in node["values"]}}


# Review kinds about a pair of lines (line A of the older report, line B of the newer one): they get the comparison table.
PAIR_REVIEW_KINDS = ("VALUE_CONFLICT", "LONG_GAP", "MAGNITUDE_JUMP", "GAP_BLOCKED", "STALE_LINK", "AMBIGUOUS_LINK",
                     "RESTATEMENT_CHECK")
# Of those, the kinds that also get the printed cell of each line as flat a_* / b_* fields (VALUE_CONFLICT has overlap_sources).
_PAIR_REF_KINDS = ("LONG_GAP", "MAGNITUDE_JUMP", "GAP_BLOCKED", "STALE_LINK", "AMBIGUOUS_LINK", "RESTATEMENT_CHECK")


def _pair_nodes(p: Dict[str, Any], dedupe_key: str, node_by: Dict[tuple, Dict[str, Any]]):
    """(line A, line B) of a pair item, each None when it is not in the current data (a stale link's old line)."""
    a_id, b_id = p.get("a_node"), p.get("b_node")
    if (a_id is None or b_id is None) and ">" in dedupe_key:
        a_id, b_id = dedupe_key.split(">", 1)
    return node_by.get((p.get("partition"), a_id)), node_by.get((p.get("partition"), b_id))


def _ref_year(p: Dict[str, Any], side: str, node: Dict[str, Any], other: Optional[Dict[str, Any]],
              status: Dict[str, str]) -> Optional[int]:
    """The year whose printed cell stands for a line in the item: the disputed year when both lines print it (a magnitude
    jump's own year, else the first MISMATCH year, else the first year both print), otherwise the line's first printed year."""
    own = p.get(f"{side}_year")
    if isinstance(own, int) and own in node["values"]:
        return own
    for want in ("MISMATCH", "MATCH"):
        for y, st in status.items():
            if st == want and int(y) in node["values"]:
                return int(y)
    return min(node["values"]) if node["values"] else None


def _attach_comparisons(review: List[Dict[str, Any]], nodes: List[Dict[str, Any]]) -> None:
    """Adds payload["comparison"] to every pair item (PAIR_REVIEW_KINDS): both lines' whole rows (all years, with their
    printed cells) side by side, and per year whether the two reports agree. Purely derived from the nodes; nothing is
    changed or re-asked. {years, conflicting_years, year_status: {year: MATCH | MISMATCH | ONLY_A | ONLY_B}, lines: [a, b]}.
    The kinds of _PAIR_REF_KINDS also get a_report, a_page_number, a_table_num, a_table_row, a_table_col (and b_*): the
    printed cell of each line in the year _ref_year picks. A side that is no longer in the data (a stale link) gets none."""
    node_by = {(n["partition"], n["id"]): n for n in nodes}
    for item in review:
        if item["kind"] not in PAIR_REVIEW_KINDS:
            continue
        p = item["payload"]
        a, b = _pair_nodes(p, item["dedupe_key"], node_by)
        status: Dict[str, str] = {}
        if a is not None and b is not None:
            ev = p.get("evidence") or series_linker.evidence(a, b)
            matches = {o["year"]: bool(o["match"]) for o in ev.get("overlap", [])}
            years = sorted(set(a["values"]) | set(b["values"]))
            for y in years:
                if y in a["values"] and y in b["values"]:
                    status[str(y)] = "MATCH" if matches.get(y, a["values"][y][0] == b["values"][y][0]) else "MISMATCH"
                else:
                    status[str(y)] = "ONLY_A" if y in a["values"] else "ONLY_B"
            p["comparison"] = {"years": years, "conflicting_years": [y for y, st in status.items() if st == "MISMATCH"],
                               "year_status": status, "lines": [_comparison_line("a", a), _comparison_line("b", b)]}
        if item["kind"] not in _PAIR_REF_KINDS:
            continue
        for side, node, other in (("a", a, b), ("b", b, a)):
            if node is None:
                continue
            year = _ref_year(p, side, node, other, status)
            p.update(series_linker.source_cell(node["report"], node["prov"].get(year) if year is not None else None, side))


def _attach_definition_changes(review: List[Dict[str, Any]], nodes: List[Dict[str, Any]],
                               compositions: List[Dict[str, Any]]) -> None:
    """For a same-name link whose values jump (VALUE_CONFLICT from a name link): when the model's breakdowns show that the
    total is made of different parts in the two reports (or was a single line and now is a group), the reason says so and
    payload["definition_change"] lists the parts. Purely derived from stored breakdowns."""
    from . import series_keys as sk
    node_by = {(n["partition"], n["id"]): n for n in nodes}
    by_cell = {}
    for c in compositions:
        by_cell.setdefault((c["fiscal_year"], c["page_number"], c["table_num"], c["total"]["row"], c["total"]["col"]), c)

    def parts_of(node):
        names, seen = [], set()
        for (pg, tn, row, col) in sorted(node.get("cells") or (), key=lambda x: tuple(v or 0 for v in x)):
            c = by_cell.get((node["report"], pg, tn, row, col))
            if not c:
                continue
            for part in c["parts"]:
                nm = part.get("field") or part.get("raw_field")
                if nm and sk.label_key(nm) not in seen:
                    seen.add(sk.label_key(nm))
                    names.append(nm)
            if names:
                break
        return names

    for item in review:
        p = item["payload"]
        if item["kind"] != "VALUE_CONFLICT" or p.get("rule") not in ("NAME_EXACT", "ALIAS_EQUIV") or ">" not in item["dedupe_key"]:
            continue
        a_id, b_id = item["dedupe_key"].split(">", 1)
        a, b = node_by.get((p["partition"], a_id)), node_by.get((p["partition"], b_id))
        if a is None or b is None:
            continue
        pa, pb = parts_of(a), parts_of(b)
        if (pa or pb) and {sk.label_key(x) for x in pa} != {sk.label_key(x) for x in pb}:
            p["definition_change"] = {"a_parts": pa, "b_parts": pb}
            before = " + ".join(pa) if pa else "a single line"
            after = " + ".join(pb) if pb else "a single line"
            p["reason"] = f"{p.get('reason') or ''}; definition changed: {before} -> {after}".lstrip("; ")


def _pair_matches(o: Dict[str, Any], p: Dict[str, Any]) -> bool:
    """A FORCE_LINK / FORCE_SPLIT override names the same two lines as a pair item (either order; the reports must match
    when the override names them)."""
    from . import series_keys as sk
    if series_linker.partition_key(o["statement_type"], o["scope"], o.get("entity_key") or "", o.get("segment_key") or "") != p.get("partition"):
        return False
    oa, ob = sk.label_key(o.get("a_label_key") or ""), sk.label_key(o.get("b_label_key") or "")
    pa, pb = sk.label_key(p.get("a_label_key") or ""), sk.label_key(p.get("b_label_key") or "")
    for x, y, rx, ry in ((pa, pb, p.get("a_report"), p.get("b_report")), (pb, pa, p.get("b_report"), p.get("a_report"))):
        if (oa, ob) == (x, y) and (not o.get("a_report") or (o["a_report"], o.get("b_report")) == (rx, ry)):
            return True
    return False


def auto_resolve(review: List[Dict[str, Any]], overrides: List[Dict[str, Any]]) -> None:
    """Marks (in place) every pair item that an existing user override already decides as resolved, with the override's id:
    a FORCE_LINK / FORCE_SPLIT on the same two lines, or a RENAME / DETACH of its line. ORPHAN_OVERRIDE items stay open."""
    from . import series_keys as sk
    for item in review:
        p = item["payload"]
        if item["kind"] not in PAIR_REVIEW_KINDS or not isinstance(p, dict):
            continue
        for o in overrides:
            if o.get("source", "user") != "user" or not o.get("active", 1):
                continue
            hit = False
            if o["kind"] in ("FORCE_LINK", "FORCE_SPLIT"):
                hit = _pair_matches(o, p)
            elif o["kind"] in ("RENAME", "DETACH"):
                part = series_linker.partition_key(o["statement_type"], o["scope"], o.get("entity_key") or "", o.get("segment_key") or "")
                a = sk.label_key(o.get("a_label_key") or "")
                hit = part == p.get("partition") and a in (sk.label_key(p.get("a_label_key") or ""), sk.label_key(p.get("b_label_key") or ""))
            if hit:
                item.update(status="resolved", resolution="override", override_id=o["id"])
                break


def _llm_overrides(links: List[Dict[str, Any]], nodes: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Every link Gemini made in this merge as an override of type LLM (a FORCE_LINK of the two lines, with their reports)."""
    first = {}
    for n in nodes:
        first.setdefault(n["partition"], n)
    out = []
    for l in links:
        n = first.get(l["partition"])
        if l.get("source") != "gemini" or n is None:
            continue
        out.append({"statement_type": n["statement_type"], "scope": n["scope"], "entity_key": n["entity_key"],
                    "segment_key": n["segment_key"], "a_label_key": l["from_label"], "a_report": l["from_report"],
                    "b_label_key": l["to_label"], "b_report": l["to_report"], "confidence": l["confidence"],
                    "link_rule": l["rule"], "note": f"Fintra AI: {l['evidence'].get('from_field')} -> {l['evidence'].get('to_field')}"})
    return out


def run_series_merge(code: str, progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
                     should_cancel: Optional[Callable[[], bool]] = None, judge=None) -> Dict[str, Any]:
    """Returns {"merged_res": <merge result>, "metrics": <relabelled metrics>}.
    Raises MergeCancelled (nothing saved) when should_cancel() turns True."""
    start = time.time()
    emit = progress_callback or (lambda _e: None)
    relabelled = database.relabel_company(code)
    metrics = database.get_all_metrics_for_company(code)
    pages_md = database.get_pages_markdown(code)
    company_name = database.get_company_info(code).get("name")
    nodes = series_linker.build_nodes(metrics, pages_md, company_name=company_name, code=code)
    overrides = database.load_merge_overrides(code, source="user")      # the LLM's own earlier links are decided again
    previous_links = database.load_active_links(code)

    # Gemini is asked once per report transition: a dry run tells how many calls to expect (an estimate).
    stats = MergeStats(nodes)
    progress = GeminiProgress(emit, plan_gemini_calls(nodes, overrides), stats)
    emit({"type": "init", "stage": "init", "percent": 5, "total_pairs": 0, "evaluated_pairs": 0, "matched_pairs": 0,
          "conflicts_count": 0, "current_statement": "", "field_a": "", "field_b": "", "status": "INITIALIZING",
          **progress.fields(),
          "subtitle": f"Linking {len(nodes)} line items across {len({n['report'] for n in nodes})} reports"
                      + (f"; about {progress.calls_total} Fintra AI calls expected" if progress.calls_total else "")})

    usage = {"prompt_tokens": 0, "candidates_tokens": 0, "total_tokens": 0, "batches_count": 0, "batch_details": [],
             "missing_answers": 0}
    if judge is None:
        judge = _default_judge(progress.on_event, should_cancel, usage)
    _check_cancelled(should_cancel)
    result = series_linker.link_company(nodes, overrides, judge=judge, on_questions=progress.round_started, on_step=progress.step)
    _check_cancelled(should_cancel)

    series, links, chain_review = series_linker.build_series(nodes, result["links"])
    review = (result["review_items"] + chain_review + _node_flag_items(nodes)
              + _stale_link_items(previous_links, links) + _orphan_override_items(overrides, nodes))
    compositions = database.get_cell_compositions(code)
    _attach_timelines(review, series, nodes)
    _attach_comparisons(review, nodes)
    _attach_definition_changes(review, nodes, compositions)
    auto_resolve(review, overrides)
    llm_overrides = _llm_overrides(links, nodes)
    conflicts = series_linker.value_conflicts_from_review(review)

    stats.finish(links, review)         # every link and every review item, those that an override resolved included
    progress.round_started("", 0)       # closes the last round: the call counts below are final
    emit({"type": "finalize", "stage": "saving", "percent": 95, "total_pairs": result["questions_total"],
          "evaluated_pairs": result["questions_total"], "matched_pairs": len(links),
          "conflicts_count": len(conflicts), "merged_clusters_count": len(series), "status": "SAVING",
          **{**progress.fields(exact=True), "round": None, "gemini_calls_total": progress.calls_done,
             "gemini_calls_done": progress.calls_done},
          "subtitle": f"{len(links)} links form {len(series)} series"})
    _check_cancelled(should_cancel)

    series_ids = database.save_series_run(code, series, links, review, llm_overrides)
    out = series_linker.series_to_rows(series, series_ids, overrides, conflicts,
                                       compositions=compositions)
    csv_text, stating_csv_text, headers = series_linker.rows_to_csv(out["rows"], out["years"])

    all_micro = [{"canonical_field": r["field"], "statement_type": r["statement_type"], **mv}
                 for r in out["rows"] for mv in r["micro_variances"].values()]
    token_usage = {
        "process_type": "MERGE", "duration_sec": round(time.time() - start, 2),
        "total_tokens": usage["total_tokens"], "prompt_tokens": usage["prompt_tokens"],
        "candidates_tokens": usage["candidates_tokens"], "total_pairs": result["questions_total"],
        "evaluated_pairs": result["questions_asked"], "batches_count": usage["batches_count"],
        "batch_details": usage["batch_details"], "missing_answers": usage["missing_answers"], "model": DEFAULT_MODEL,
    }
    rule_counts: Dict[str, int] = {}
    for l in links:
        rule_counts[l["rule"]] = rule_counts.get(l["rule"], 0) + 1
    merged_res = {
        "headers": headers, "years": out["years"], "rows": out["rows"],
        "csv_text": csv_text, "stating_csv_text": stating_csv_text,
        "merge_report": out["merge_report"],
        "conflicts": conflicts, "value_mismatch_conflicts": conflicts, "value_mismatch_count": len(conflicts),
        "micro_variance_merges": all_micro, "micro_variances_count": len(all_micro),
        "total_pairs": result["questions_total"], "evaluated_pairs": result["questions_asked"],
        "matched_pairs": len(links), "token_usage": token_usage,
        # additive series summary
        "series_count": len(series), "links_by_rule": rule_counts,
        "review_open_count": len(database.list_review_items(code, "open")),
        "relabelled": relabelled,
        "merge_stats": progress.fields(exact=True)["merge_stats"],
    }
    return {"merged_res": merged_res, "metrics": metrics}


__all__ = ["run_series_merge", "MergeCancelled"]
