"""Series linking: the core of the multi-year merge (pure functions, no database, no AI calls).

A *node* is one line item in one annual report (statement, scope, entity/segment, section, label).
A *series* is a chain of nodes, one per report, linked report by report. Each report N is linked to
the next report N+1 one-to-one, using in order of strength:

  R0  user FORCE_LINK overrides
  R1  the same normalised label (or a user alias)                       -> NAME_EXACT / ALIAS_EQUIV
  R2  the comparative-year value is identical and unique both ways      -> VALUE_UNIQUE
  R2s split / merge detection (values summing up)                       -> review item, no link
  R3  ambiguous cases, judged by Gemini (injected `judge`)               -> GEMINI
  R4  gap linking for reports or years that are missing (name or value) -> GAP_NAME / GAP_VALUE

Hard vetoes (never linked unless a user forces it): different value kind or unit, accounting
contrasts (current vs non-current, ...), the two labels printed side by side in the same report,
opening vs closing balances, contrasting sections, user FORCE_SPLIT. Different statements, scopes,
entities or segments are separate partitions and are never linked at all.
"""

import itertools
import os
import re
from collections import Counter, defaultdict
from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple

from . import series_keys as sk
from .csv_service import (STATEMENT_TYPE_ORDER, csv_safe_row, extract_year_int, format_numeric_value,
                          is_distinct_subsidiary_entity, normalize_entity_name, normalize_statement_type)
from .merge_service import are_candidate_pair, has_unmergable_keyword_contrast, normalize_text

RULES_VERSION = 2

# How many reports (in company order) a line may skip and still be continued; env MERGE_MAX_GAP_REPORTS.
MAX_GAP_REPORTS = max(0, int(os.environ.get("MERGE_MAX_GAP_REPORTS", "3") or 3))
LONG_GAP_CONFIDENCE = 0.75

# Gemini's answer is a verdict (SAME links, UNSURE goes to review, DIFFERENT is a no); see merge_service.VERDICT_CONFIDENCE
MAX_CANDIDATES = 5
BALANCE_TYPES = {"ASSET", "LIABILITY", "EQUITY"}
JOIN_MAX_SPAN = 8          # R5: how many reports apart two series may be and still be offered to Gemini
JOIN_MIN_SUPPORT = 0.6     # R5: minimum name similarity

Judge = Callable[[List[Dict[str, Any]]], Dict[int, Dict[str, Any]]]


# ---------------------------------------------------------------------------
# Nodes
# ---------------------------------------------------------------------------
def partition_key(stmt: str, scope: str, entity_key: str, segment_key: str) -> str:
    return f"{stmt}|{scope}|{entity_key}|{segment_key}"


def node_id(report: str, label: str, section: str, side: str = "") -> str:
    return f"{report}|{label}|{section}" + (f"|{side}" if side else "")


def _parent_candidates(metrics: List[Dict[str, Any]], company_name: Optional[str], code: Optional[str]) -> set:
    """The reporting company, as the user selected it (company name and security code); nothing is guessed
    from the data."""
    return {c for c in (company_name, code) if c}


def _category_key(category: Optional[str]) -> str:
    """The heading a line sits under, without its " - <entity>" suffix, as a label key."""
    return sk.label_key(re.sub(r"\s-\s.*$", "", str(category or "")))


def _identity(m: Dict[str, Any], parents: set) -> Optional[Dict[str, Any]]:
    """What decides the node a metric belongs to (None for a metric that never takes part in the merge)."""
    if m.get("verification_flag") == "FIELD_UNIDENTIFIED":
        return None      # reported to the user (series_merge), never merged
    stmt = normalize_statement_type(m.get("statement_type"))
    scope = str(m.get("scope") or sk.CONSOLIDATED).upper()
    scope = sk.STANDALONE if ("STANDALONE" in scope or "SOLO" in scope) else sk.CONSOLIDATED
    entity = m.get("entity")
    entity_key = normalize_entity_name(entity) if entity else ""     # the reporting company's own entity is kept too
    shown_entity = entity if entity_key and is_distinct_subsidiary_entity(entity, parents) else None
    segment_key = normalize_text(m.get("segment") or "")
    field = m.get("field") or m.get("concept_name") or m.get("raw_field") or ""
    label = sk.label_key(field)
    if not label:
        return None
    section = sk.section_key(m.get("category"), stmt)
    return {"stmt": stmt, "scope": scope, "entity_key": entity_key, "shown_entity": shown_entity,
            "segment_key": segment_key, "field": field, "label": label, "section": section,
            "side": sk.parent_side(m.get("category"), section),
            "pkey": partition_key(stmt, scope, entity_key, segment_key),
            "base": (m["fiscal_year"], label, section), "cat": _category_key(m.get("category")),
            "raw_key": sk.label_key(m.get("raw_field") or field)}


def build_nodes(metrics: List[Dict[str, Any]], pages_md: Dict[Tuple[str, int], str],
                company_name: Optional[str] = None, code: Optional[str] = None) -> List[Dict[str, Any]]:
    """One node per (report, statement, scope, entity, segment, section, label) from stored metrics
    (the shape returned by database.get_all_metrics_for_company, after database.relabel_company).

    Two printed lines that share label and section but print different values for the same year (the
    same line name under two headings, or the rows of a record list that all got one concept name) are
    told apart: the node id gets the heading (`|c:<heading>`) and, inside one heading, the printed row
    label (`|r:<row label>`). Nodes that never collide keep the plain id."""
    nodes = _build_nodes(metrics, pages_md, company_name, code, None)
    clash = {(n["partition"], n["report"], n["label"], n["section"], n["side"]) for n in nodes
             if {"INTRA_NODE_CONFLICT", "PERIOD_COLLISION"} & set(n["flags"])}
    return _build_nodes(metrics, pages_md, company_name, code, clash) if clash else nodes


def _build_nodes(metrics: List[Dict[str, Any]], pages_md: Dict[Tuple[str, int], str],
                 company_name: Optional[str], code: Optional[str], split: Optional[set]) -> List[Dict[str, Any]]:
    parents = _parent_candidates(metrics, company_name, code)
    facets: Dict[Tuple, List[Tuple[str, str]]] = defaultdict(list)
    if split:
        for m in metrics:
            ident = _identity(m, parents)
            key = ident and (ident["pkey"], *ident["base"], ident["side"])
            if key and (key[0], key[1], key[2], key[3], key[4]) in split:
                facets[key].append((ident["cat"], ident["raw_key"]))
    headers_cache: Dict[Tuple[str, int], Dict[Tuple[int, int], str]] = {}
    max_row: Dict[Tuple[str, int, int], int] = defaultdict(int)
    for m in metrics:
        for o in m.get("data") or []:
            k = (m["fiscal_year"], m["page_number"], o.get("table_num") or 1)
            max_row[k] = max(max_row[k], o.get("table_row") or 0)

    nodes: Dict[Tuple[str, str], Dict[str, Any]] = {}
    for m in metrics:
        ident = _identity(m, parents)
        if ident is None:
            continue
        report = m["fiscal_year"]
        stmt, scope, entity_key, segment_key = ident["stmt"], ident["scope"], ident["entity_key"], ident["segment_key"]
        shown_entity, field, label, section = ident["shown_entity"], ident["field"], ident["label"], ident["section"]
        pkey, side = ident["pkey"], ident["side"]
        disc = ""
        group = facets.get((pkey, report, label, section, side))
        if group:
            if len({c for c, _ in group}) > 1:
                disc = f"c:{ident['cat']}"
            if len({r for c, r in group if c == ident["cat"]}) > 1:
                disc += ("|" if disc else "") + f"r:{ident['raw_key']}"
        data = [o for o in m.get("data") or [] if o.get("numeric_value") is not None]
        units = [o.get("unit") for o in data]
        kind = m.get("metric_type") if m.get("metric_type") in sk.VALUE_KINDS else sk.value_kind(
            f"{m.get('raw_field') or ''} {field}", units, m.get("metric_type"),
            [o.get("numeric_value") for o in data])
        unit = sk.unit_for_kind(kind, units)
        nid = node_id(report, label, section, side) + (f"|{disc}" if disc else "")
        node = nodes.get((pkey, nid))
        if node is None:
            node = nodes[(pkey, nid)] = {
                "id": nid, "partition": pkey, "report": report, "report_end": sk.report_end_year(report),
                "statement_type": stmt, "scope": scope, "entity": shown_entity,
                "entity_key": entity_key, "segment": m.get("segment") or None, "segment_key": segment_key,
                "section": section, "side": side, "label": label, "core": sk.core_key(label), "cat": ident["cat"], "disc": disc, "fields": [], "raw_labels": [], "raw_label_keys": set(),
                "kind": kind, "unit": unit, "temporal": m.get("temporal_state") or "PERIOD_FLOW",
                "aggregation_level": m.get("aggregation_level") or "LINE_ITEM",
                "category": m.get("category") or "", "tag": m.get("tag") or "FINANCIAL_METRIC",
                "concept_name": m.get("concept_name") or field,
                "action": m.get("action"), "qualifier": m.get("qualifier"),
                "values": {}, "prov": {}, "alt_periods": {}, "flags": [], "rank": 1.0, "_total": {}, "cells": set(),
                "prints": {}, "_restated": {},
            }
        if field not in node["fields"]:
            node["fields"].append(field)
        if m.get("verification_flag") == "LABEL_EVIDENCE_FAILED":
            node["flags"].append("LABEL_EVIDENCE_FAILED")
        raw = m.get("raw_field") or field
        if raw not in node["raw_labels"]:
            node["raw_labels"].append(raw)
            raw_key = sk.label_key(raw)
            if _informative_label(raw_key):
                node["raw_label_keys"].add(raw_key)

        page_key = (report, m["page_number"])
        if page_key not in headers_cache:
            headers_cache[page_key] = sk.column_headers(pages_md.get(page_key, ""))
        for o in data:
            year = o.get("year") if isinstance(o.get("year"), int) else extract_year_int(o.get("year"))
            if year is None:
                continue
            mult = float(o.get("multiplier") or 1.0)
            value = float(o["numeric_value"]) * mult
            t_num = o.get("table_num") or 1
            node["cells"].add((m["page_number"], t_num, o.get("table_row"), o.get("table_col")))
            months = sk.period_months(headers_cache[page_key].get((t_num, o.get("table_col") or 0), ""))
            prov = {"metric_id": m.get("id"), "report_year": report, "page_number": m["page_number"],
                    "table_num": t_num, "table_row": o.get("table_row"), "table_col": o.get("table_col"),
                    "field": field, "raw_field": raw, "months": months,
                    "date": o.get("date"), "restated": bool(o.get("restated"))}
            restated = bool(o.get("restated"))
            if months not in (None, 12):
                node["alt_periods"].setdefault(year, []).append({"months": months, "value": value, **prov})
                continue
            is_total = _is_summary_row(m, raw)
            if year in node["values"] and restated != node["_restated"].get(year, False):
                old_prov = node["prov"][year]
                if ((old_prov.get("page_number"), old_prov.get("table_num"), old_prov.get("table_row")) ==
                        (m["page_number"], t_num, o.get("table_row")) and old_prov.get("table_col") != o.get("table_col")):
                    # the same printed row has the year twice, once as printed and once restated: the restated figure
                    # is the year's value and the other print stays a disclosure (it is no conflict, no new line)
                    old = (node["values"][year][0], node["values"][year][1], old_prov)
                    mine = (value, mult, prov)
                    plain, winner = (old, mine) if restated else (mine, old)
                    node["prints"].setdefault(year, []).append(plain)
                    node["values"][year] = (winner[0], winner[1])
                    node["prov"][year] = winner[2]
                    node["_restated"][year] = True
                    continue
            if year in node["values"]:
                old_value = node["values"][year][0]
                old_prov = node["prov"][year]
                other_row = (old_prov.get("page_number"), old_prov.get("table_num"), old_prov.get("table_row")) != \
                    (m["page_number"], t_num, o.get("table_row"))
                if other_row and sk.agree(old_value, value, kind, node["values"][year][1], mult):
                    # the same report printing the line again with the same figure is fine (not a restatement);
                    # with a different figure it is a conflict below (review)
                    continue
                if not sk.agree(old_value, value, kind, node["values"][year][1], mult):
                    old_prov = node["prov"][year]
                    same_col = old_prov.get("table_col") == o.get("table_col")
                    same_table = (old_prov.get("page_number"), old_prov.get("table_num")) == (m["page_number"], t_num)
                    if same_table and same_col and is_total and not node["_total"].get(year):
                        # a breakdown (In Bangladesh / Outside Bangladesh ...) shares the concept name with its
                        # own total row: the total is the line's value (the parts stay visible in cell-details)
                        node["values"][year] = (value, mult)
                        node["prov"][year] = prov
                        node["_total"][year] = True
                        continue
                    node["flags"].append("INTRA_NODE_CONFLICT" if same_col else "PERIOD_COLLISION")
                    node["alt_periods"].setdefault(year, []).append({"months": months, "value": value, **prov})
                continue
            node["values"][year] = (value, mult)
            node["prov"][year] = prov
            node["_restated"][year] = restated
            node["_total"][year] = is_total
            mr = max_row.get((report, m["page_number"], t_num)) or 0
            if mr and o.get("table_row"):
                node["rank"] = min(node["rank"], o["table_row"] / mr)
    out = list(nodes.values())
    for n in out:
        n["flags"] = sorted(set(n["flags"]))
    return out


_GENERIC_LABEL_WORDS = {"total", "sub", "subtotal", "grand", "other", "others", "net", "balance", "amount", "taka", "bdt"}


def _is_summary_row(m: Dict[str, Any], raw_label: str) -> bool:
    """A subtotal / grand-total row (by the model's level, or a printed "Total ..." label)."""
    return (m.get("aggregation_level") in ("SUBTOTAL", "GRAND_TOTAL")
            or bool(re.search(r"\btotal\b", str(raw_label or ""), re.I)))


def _informative_label(key: str) -> bool:
    """A printed label specific enough to identify a line on its own ("Total" or "Others" is not)."""
    words = key.split()
    return len(words) >= 2 and not set(words) <= _GENERIC_LABEL_WORDS


def source_cell(report: str, prov: Optional[Dict[str, Any]], prefix: str) -> Dict[str, Any]:
    """The printed cell a value came from, as {<prefix>_report, <prefix>_page_number, <prefix>_table_num,
    <prefix>_table_row, <prefix>_table_col} (None for unknown parts)."""
    prov = prov or {}
    return {f"{prefix}_report": report, f"{prefix}_page_number": prov.get("page_number"),
            f"{prefix}_table_num": prov.get("table_num"), f"{prefix}_table_row": prov.get("table_row"),
            f"{prefix}_table_col": prov.get("table_col")}


def _overlap_sources(h: Dict[str, Any], n: Dict[str, Any], overlap: List[Dict[str, Any]]) -> Dict[str, Any]:
    """{year: {a_*, b_*}}: the printed cells of both sides for every overlapping year."""
    return {str(o["year"]): {**source_cell(h["report"], h["prov"].get(o["year"]), "a"),
                             **source_cell(n["report"], n["prov"].get(o["year"]), "b")} for o in overlap}


def _display(node: Dict[str, Any]) -> str:
    return node["fields"][0] if node["fields"] else node["label"]


def _disc_clash(h: Dict[str, Any], n: Dict[str, Any]) -> bool:
    """Two lines of one report that were told apart by heading / row label (node `disc`) are different lines."""
    return bool(h.get("disc") and n.get("disc") and h["disc"] != n["disc"])


def _shown(node: Dict[str, Any]) -> str:
    """The name Gemini sees: the concept name, plus the heading and the printed row label when the node was
    told apart from a same-named line of its report."""
    name = _display(node)
    if not node.get("disc"):
        return name
    raw = node["raw_labels"][0] if node.get("raw_labels") else ""
    return f"{name} [heading: {node.get('category') or '-'}; row: {raw or '-'}]"


# ---------------------------------------------------------------------------
# Overrides
# ---------------------------------------------------------------------------
def _override_partition(o: Dict[str, Any]) -> str:
    return partition_key(o["statement_type"], o["scope"], o.get("entity_key") or "", o.get("segment_key") or "")


class _Overrides:
    """User decisions resolved against one partition's labels."""

    def __init__(self, overrides: Iterable[Dict[str, Any]], pkey: str):
        self.force_link_edges: List[Dict[str, Any]] = []
        self.alias_parent: Dict[str, str] = {}
        self.split_pairs: Set[frozenset] = set()
        self.split_edges: Set[Tuple[str, str, str, str]] = set()
        self.detached: Set[Tuple[str, str]] = set()
        self.renames: Dict[str, str] = {}
        for o in sorted(overrides, key=lambda x: (x.get("created_at") or "", x.get("id") or 0)):
            if _override_partition(o) != pkey:
                continue
            a = sk.label_key(o.get("a_label_key") or "")
            b = sk.label_key(o.get("b_label_key") or "")
            kind = o["kind"]
            if kind == "FORCE_LINK" and a and b:
                if o.get("a_report") and o.get("b_report"):
                    self.force_link_edges.append({**o, "a": a, "b": b})
                else:
                    self._union(a, b)
            elif kind == "FORCE_SPLIT" and a and b:
                if o.get("a_report") and o.get("b_report"):
                    self.split_edges.add((o["a_report"], a, o["b_report"], b))
                    self.split_edges.add((o["b_report"], b, o["a_report"], a))
                else:
                    self.split_pairs.add(frozenset((a, b)))
            elif kind == "DETACH" and a:
                self.detached.add((o.get("a_report") or "*", a))
            elif kind == "RENAME" and a and o.get("new_label"):
                self.renames[a] = o["new_label"]

    def _find(self, x: str) -> str:
        while self.alias_parent.get(x, x) != x:
            x = self.alias_parent[x]
        return x

    def _union(self, a: str, b: str) -> None:
        ra, rb = self._find(a), self._find(b)
        if ra != rb:
            self.alias_parent[rb] = ra

    def alias_equal(self, a: str, b: str) -> bool:
        return a == b or self._find(a) == self._find(b)

    def is_split(self, h: Dict[str, Any], n: Dict[str, Any]) -> bool:
        return (frozenset((h["label"], n["label"])) in self.split_pairs
                or (h["report"], h["label"], n["report"], n["label"]) in self.split_edges)

    def is_detached(self, n: Dict[str, Any]) -> bool:
        return ("*", n["label"]) in self.detached or (n["report"], n["label"]) in self.detached


# ---------------------------------------------------------------------------
# Evidence and vetoes
# ---------------------------------------------------------------------------
def evidence(h: Dict[str, Any], n: Dict[str, Any]) -> Dict[str, Any]:
    return sk.compare_values(h["values"], n["values"], h["kind"])


def evidence_token(ev: Dict[str, Any]) -> str:
    ov = ev["overlap"]
    if not ov:
        return "∅"
    o = max(ov, key=lambda x: x["year"])
    if o["match"]:
        return f"={o['year']}:{o['a']:,.2f}".rstrip("0").rstrip(".")
    base = o["a"] if o["a"] else o["b"]
    pct = ((o["b"] - o["a"]) / abs(base) * 100) if base else 0.0
    return f"≠{o['year']}:{pct:+.1f}%"


def veto(h: Dict[str, Any], n: Dict[str, Any], coexist: Callable[[str, str], bool], ov: _Overrides) -> Optional[str]:
    if ov.is_split(h, n):
        return "user FORCE_SPLIT"
    if _disc_clash(h, n):
        return "two different lines of one report (heading / row label differ)"
    if h["kind"] != n["kind"]:
        return f"value kind {h['kind']} vs {n['kind']}"
    if h["unit"] != n["unit"]:
        return f"unit {h['unit']} vs {n['unit']}"
    if h["label"] != n["label"] and coexist(h["label"], n["label"]):
        return "both printed in the same report"
    if sk.sections_contrast(h["section"], n["section"]):
        return f"section {h['section']} vs {n['section']}"
    if _side_clash(h, n):
        return "parent heading is about assets in one report and liabilities in the other"
    t1, t2 = str(h["temporal"]), str(n["temporal"])
    if ("OPENING" in t1 and "CLOSING" in t2) or ("OPENING" in t2 and "CLOSING" in t1):
        return "opening vs closing balance"
    if h["label"] != n["label"] and h["core"] != n["core"]:      # "X" vs "X total": the coexistence check above guards it
        bad, reason = has_unmergable_keyword_contrast(_display(h), _display(n))
        if bad and not (reason.startswith(_ENTITY_REASONS) and h["raw_label_keys"] & n["raw_label_keys"]):
            return reason       # an entity name in only one of the two names is no contrast when both print the same row label
    return None


_ENTITY_REASONS = ("Entity/segment breakdown", "Distinct entity or segment breakdown")
# veto() reasons about an entity word in only one of two names ("... Associate Companies" vs "... Associate and Subsidiary Companies",
# "owner of company" vs "owner of parent"). The keyword contrast is the LAST check of veto(), so such a reason means no hard veto applies.
# These pairs are not linked by name or value (R1, R2, R4 keep the full veto), but they ARE shown to Fintra AI when the figures do not
# disagree, which decides whether the entity word is a different account or only a wording.
SOFT_VETO_REASONS = ("Entity scope contrast", "Entity/segment breakdown", "Distinct entity or segment breakdown")


def _soft_veto(reason: Optional[str]) -> bool:
    return bool(reason) and str(reason).startswith(SOFT_VETO_REASONS)
_HEADING_STOP = {"consolidated", "standalone", "separate", "note", "notes", "a", "b", "the", "details", "total",
                 "statement", "schedule"}
MAGNITUDE_JUMP_RATIO = 50.0


def _heading_tokens(node: Dict[str, Any]) -> set:
    return {t for t in sk.stem_tokens(re.sub(r"\d+(\.\d+)*", " ", node.get("cat") or ""))
            if len(t) > 1 and t not in _HEADING_STOP}


def _heading_compatible(h: Dict[str, Any], n: Dict[str, Any]) -> bool:
    """False only when both lines sit under a known heading and the two headings share no informative word
    (note numbers and words like "consolidated" or "note" do not count)."""
    a, b = _heading_tokens(h), _heading_tokens(n)
    return not a or not b or bool(a & b)


def _magnitude_ratio(h: Dict[str, Any], n: Dict[str, Any]) -> Optional[Tuple[float, int, int]]:
    """(ratio, year a, year b): the larger over the smaller absolute value in the two nearest years (sign ignored)."""
    pairs = [(abs(y1 - y2), y1, y2) for y1 in h["values"] for y2 in n["values"]]
    if not pairs:
        return None
    _, y1, y2 = min(pairs)
    a, b = abs(h["values"][y1][0]), abs(n["values"][y2][0])
    return (max(a, b) / min(a, b), y1, y2) if a and b else None


def _side_clash(h: Dict[str, Any], n: Dict[str, Any]) -> bool:
    return bool(h.get("side") and n.get("side") and h["side"] != n["side"])


def _name_match(h: Dict[str, Any], n: Dict[str, Any], ov: "_Overrides") -> bool:
    """Same label key (also without the word "total"), same informative printed label, or a user alias (never across value kinds)."""
    if h["kind"] != n["kind"] or ov.is_split(h, n):
        return False
    if _disc_clash(h, n):
        return False        # two lines of one report told apart by heading / row label stay apart
    return (ov.alias_equal(h["label"], n["label"]) or bool(h["raw_label_keys"] & n["raw_label_keys"])
            or (bool(h["core"]) and h["core"] == n["core"]))      # "X" and "X total": the coexistence veto keeps a part apart from its subtotal


def _lexical_support(h: Dict[str, Any], n: Dict[str, Any]) -> float:
    a, b = _display(h), _display(n)
    sim = max(sk.name_similarity(a, b), sk.name_similarity(h["label"], n["label"]))
    if are_candidate_pair(a, b)[0]:
        sim = max(sim, 0.65)
    return sim


NAME_DOUBT_CHANGE = 0.50        # a name link whose values move more than this in a common year is shown for review
NAME_DOUBT_CONFIDENCE = 0.8


def value_change(ev: Dict[str, Any], statement_type: str = "") -> Tuple[float, Optional[int], Optional[int]]:
    """(largest relative change, its year, a balance-sheet sign-flip year or None) over the common years whose values disagree.
    A change is |a - b| / max(|a|, |b|); a small one is a restatement, one above NAME_DOUBT_CHANGE is a hard conflict."""
    worst, year, flip = 0.0, None, None
    for o in ev.get("overlap") or []:
        if o.get("match"):
            continue
        a, b = o["a"], o["b"]
        big = max(abs(a), abs(b))
        change = abs(a - b) / big if big else 0.0
        if change > worst:
            worst, year = change, o["year"]
        if statement_type == "BALANCE_SHEET" and a and b and (a > 0) != (b > 0) and flip is None:
            flip = o["year"]
    return worst, year, flip


def high_variance(ev: Dict[str, Any]) -> bool:
    """True when, in a common year, the newer report's figure differs from the older one's by more than 100 % of the older figure
    (a RESTATED_HIGH variance: a figure that was 0 and is not any more counts). Smaller differences are restatements that need no review."""
    for o in ev.get("overlap") or []:
        if o.get("match"):
            continue
        a, b = o["a"], o["b"]
        if abs(b - a) > 1e-4 and (not a or abs(b - a) / abs(a) > sk.RESTATEMENT_MID_MAX / 100.0):
            return True
    return False


def _doubt_needs_review(doubt: Optional[str], ev: Dict[str, Any]) -> bool:
    """A name link's doubt becomes a review item when it is about more than the figures (a sign flip, headings that share no word),
    or when the figures differ by a HIGH variance."""
    if not doubt:
        return False
    parts = [p.strip() for p in doubt.split(";") if p.strip()]
    if any(not p.startswith("values differ") for p in parts):
        return True
    return high_variance(ev)


# How much attention a review item needs (derived when the items are listed, so old items get it too).
# HIGH: figures that may be wrong in the merged table. MEDIUM: a link is missing, doubtful or a structure changed.
# LOW: a link was made with Gemini's help and only a small difference (a likely restatement) is left to confirm.
ATTENTION = {
    "HIGH": ["MAGNITUDE_JUMP", "INTRA_NODE_CONFLICT", "PERIOD_COLLISION"],
    "MEDIUM": ["AMBIGUOUS_LINK", "STALE_LINK", "LONG_GAP", "GAP_BLOCKED", "SPLIT_DETECTED", "MERGE_DETECTED",
               "SERIES_INCONSISTENT", "FIELD_UNIDENTIFIED", "ORPHAN_OVERRIDE"],
    "LOW": ["RESTATEMENT_CHECK", "LABEL_EVIDENCE_FAILED"],
}
ATTENTION_RANK = {"HIGH": 1, "MEDIUM": 2, "LOW": 3}


LOW_PRIORITY_STATEMENTS = ("NOTES_AND_FOOTNOTES", "CHANGES_IN_EQUITY")


def review_statement(payload: Dict[str, Any]) -> str:
    """The statement type an item is about: its `statement_type`, else the one in its partition key ("" when it names none)."""
    return payload.get("statement_type") or str(payload.get("partition") or "").split("|")[0]


def review_attention(kind: str, payload: Dict[str, Any]) -> str:
    """HIGH / MEDIUM / LOW for a review item. An item about a note (NOTES_AND_FOOTNOTES) or the statement of changes in
    equity is always LOW, whatever its kind. A VALUE_CONFLICT is HIGH when the figures differ by more than
    NAME_DOUBT_CHANGE, a balance-sheet line flips sign, or the pair was not linked because another line matched
    exactly; a conflict that is only about the headings is MEDIUM."""
    if review_statement(payload) in LOW_PRIORITY_STATEMENTS:
        return "LOW"
    if kind == "VALUE_CONFLICT":
        worst, _, flip = value_change(payload.get("evidence") or {}, payload.get("statement_type") or "")
        return "HIGH" if payload.get("rejected") or flip is not None or worst > NAME_DOUBT_CHANGE else "MEDIUM"
    for level, kinds in ATTENTION.items():
        if kind in kinds:
            return level
    return "MEDIUM"


def _name_link_doubt(h: Dict[str, Any], n: Dict[str, Any], ev: Dict[str, Any]) -> Optional[str]:
    """Why a same-name link deserves a look, or None. Small differences are restatements and stay silent:
    only a large change, a sign flip on a balance-sheet line, or two headings with nothing in common count.
    Only links whose values disagree are checked: agreeing values prove the link."""
    if ev["class"] != sk.DISAGREE:
        return None
    reasons = []
    worst = None
    for o in ev["overlap"]:
        if o["match"]:
            continue
        a, b = o["a"], o["b"]
        big = max(abs(a), abs(b))
        change = abs(a - b) / big if big else 0.0
        if change > NAME_DOUBT_CHANGE and (worst is None or change > worst[0]):
            worst = (change, o["year"])
        if h["statement_type"] == "BALANCE_SHEET" and a and b and (a > 0) != (b > 0):
            reasons.append(f"sign flip in {o['year']}")
    if worst:
        reasons.insert(0, f"values differ by {round(worst[0] * 100)} % in {worst[1]}")
    if h.get("cat") and n.get("cat") and not _heading_compatible(h, n):
        reasons.append(f"headings share no word: '{h['cat']}' vs '{n['cat']}'")
    return "; ".join(reasons) or None


def _value_match_instead_of_name(h, n, heads, new_nodes, m_new, st):
    """For a same-name link whose values moved a lot: the one other unmatched line of the next report whose values
    match the old line exactly, when nobody else could claim it (no head has its name, no other head matches its
    values). None when there is no such single line."""
    ov = st["ov"]
    never = lambda a, b: False      # noqa: E731  an exact value match outweighs "both names are printed in some report"
    head_labels = {x["label"] for x in heads}
    found = []
    for m in new_nodes:
        if m["id"] in m_new or m["id"] == n["id"] or m["label"] in head_labels or m["kind"] != h["kind"]:
            continue
        if veto(h, m, never, ov):
            continue
        if evidence(h, m)["class"] == sk.AGREE:
            found.append(m)
    if len(found) != 1:
        return None
    m = found[0]
    rivals = [x for x in heads if x["id"] != h["id"] and not veto(x, m, never, ov) and evidence(x, m)["class"] == sk.AGREE]
    return None if rivals else m


def _value_link_supported(h: Dict[str, Any], n: Dict[str, Any]) -> bool:
    """A unique value match links on its own only inside the same known section, or with clear name
    support; otherwise Gemini decides (values can coincide across unrelated lines)."""
    if h["section"] == n["section"] and h["section"] != "OTHER":
        return True
    return _lexical_support(h, n) >= 0.5


def _score(h: Dict[str, Any], n: Dict[str, Any], ev: Dict[str, Any]) -> float:
    value_score = {sk.AGREE: 1.0, sk.NONE: 0.3, sk.DISAGREE: 0.0}[ev["class"]]
    section_eq = 1.0 if h["section"] == n["section"] and h["section"] != "OTHER" else 0.0
    position = 1.0 - abs(h["rank"] - n["rank"])
    return 0.5 * value_score + 0.3 * _lexical_support(h, n) + 0.1 * section_eq + 0.1 * position


# ---------------------------------------------------------------------------
# Linking
# ---------------------------------------------------------------------------
def _link(h, n, rule, source, conf, ev, **extra) -> Dict[str, Any]:
    flips = [o["year"] for o in ev["overlap"] if o.get("sign_flip")]
    if flips:
        extra["sign_convention_changed"] = flips     # same size, opposite sign: printed as a subtraction in one report
    return {
        "partition": h["partition"], "from_node": h["id"], "to_node": n["id"],
        "from_report": h["report"], "to_report": n["report"],
        "from_label": h["label"], "to_label": n["label"],
        "rule": rule, "source": source, "confidence": round(conf, 4),
        "gap": extra.pop("gap", 0),  # reports skipped between the two lines (company order)
        "evidence": {"class": ev["class"], "overlap": ev["overlap"], "weight": ev["weight"],
                     "from_field": _display(h), "to_field": _display(n),
                     "sections": [h["section"], n["section"]], **extra},
    }


JOIN_ROUND = "JOIN"      # round label of the final join pass (the others are report labels)


def _resolve_questions(questions, judge, on_questions=None, round_label=""):
    """Asks `judge` (one batched call) every question, every time: nothing Gemini answers is remembered between
    merges or between report transitions. on_questions(round_label, number of questions) is called just before the
    judge is. Returns (resolved by qid, number of questions sent)."""
    resolved = {}
    asked = 0
    if questions and judge is not None:
        if on_questions:
            on_questions(round_label, len(questions))
        answers = judge(questions) or {}
        asked = len(questions)
        for q in questions:
            a = answers.get(q["qid"])
            if a is not None:
                resolved[q["qid"]] = a
    return resolved, asked


def _join_questions(states, links, rank):
    """R5: lines whose series stopped and lines that start a series later in the same partition, with
    similar names. Candidates for one batched Gemini call (see link_company)."""
    questions = []
    out_from = {(l["partition"], l["from_node"]) for l in links}
    in_to = {(l["partition"], l["to_node"]) for l in links}
    for pkey, st in states.items():
        nodes = [n for ns in st["by_report"].values() for n in ns]
        tails = [n for n in nodes if (pkey, n["id"]) not in out_from]
        starts = [n for n in nodes if (pkey, n["id"]) not in in_to]
        for t in sorted(tails, key=lambda x: (rank[x["report"]], x["label"])):
            cands = []
            for s_ in starts:
                span = rank[s_["report"]] - rank[t["report"]]
                if span < 1 or span > JOIN_MAX_SPAN:
                    continue
                blocked = veto(t, s_, st["coexist"], st["ov"])
                if blocked and not _soft_veto(blocked):
                    continue
                if not (_informative_label(t["label"]) and _informative_label(s_["label"])):
                    continue          # "Subtotal" / "Total" alone cannot identify a line across years
                support = _lexical_support(t, s_)
                if support < JOIN_MIN_SUPPORT and not are_candidate_pair(_display(t), _display(s_))[0]:
                    continue
                ev = evidence(t, s_)
                if ev["class"] == sk.DISAGREE:
                    continue
                cands.append((support - 0.01 * span, s_, ev))
            if not cands:
                continue
            cands.sort(key=lambda x: (-x[0], x[1]["label"]))
            cands = cands[:MAX_CANDIDATES]
            questions.append({
                "qid": next(_QID), "partition": pkey, "statement_type": t["statement_type"],
                "head_node": t, "head_key": t["label"],
                "head": {"label": _shown(t), "report": t["report"], "section": t["section"]},
                "candidates": [{"label": _shown(n), "key": n["label"], "report": n["report"], "section": n["section"],
                                "evidence": evidence_token(ev), "class": ev["class"], "node": n, "ev": ev}
                               for _, n, ev in cands],
                "unique_agree": False,
            })
    return questions


def _join_series(states, links, rank, judge, on_questions):
    """R5, after every report is linked: one batched Gemini pass over lines whose series stopped and
    lines that start a series later and look like the same line (for example a line the model named
    differently for a few reports)."""
    questions = _join_questions(states, links, rank)
    resolved, n_asked = _resolve_questions(questions, judge, on_questions, JOIN_ROUND)
    out_links, review = [], []
    rnd = {"questions": questions, "matched_heads": set(), "matched_new": set()}
    acc = _accept_answers(rnd, resolved)
    for l in acc["links"]:
        l["rule"] = "GEMINI_JOIN"
        l["gap"] = rank[l["to_report"]] - rank[l["from_report"]] - 1
        out_links.append(l)
        if l["gap"] >= 2:
            review.append({"kind": "LONG_GAP", "dedupe_key": f"{l['from_node']}>{l['to_node']}",
                           "payload": {"partition": l["partition"], "a_label_key": l["from_label"], "a_report": l["from_report"],
                                       "a_field": l["evidence"]["from_field"], "b_label_key": l["to_label"],
                                       "b_report": l["to_report"], "b_field": l["evidence"]["to_field"],
                                       "a_node": l["from_node"], "b_node": l["to_node"],
                                       "reports_skipped": l["gap"], "decided_by": "gemini"}})
    review.extend(acc["review"])
    return {"links": out_links, "review": review, "asked": n_asked, "total": len(questions)}


def link_company(nodes: List[Dict[str, Any]], overrides: List[Dict[str, Any]],
                 judge: Optional[Judge] = None,
                 on_questions: Optional[Callable[[str, int], None]] = None,
                 max_gap: Optional[int] = None,
                 on_step: Optional[Callable[[int, int, List[Dict[str, Any]], List[Dict[str, Any]]], None]] = None) -> Dict[str, Any]:
    """Links every partition report by report, nearest report first.

    max_gap: how many reports a line may skip and still be linked (default MAX_GAP_REPORTS); reports
    are counted in company order. Skipping 1 needs a name match or a unique value; skipping 2+ needs
    a name match and is flagged for review (LONG_GAP).

    judge: callable(questions) -> {qid: {"choice", "confidence", "type_head", "type_choice"}}; None = never ask.
    on_questions(round, n): called before each judge call; round is the report whose lines are being linked, or
    JOIN_ROUND for the final join pass (progress reporting).
    on_step(reports_done, reports_total, links, review): called when a report is fully linked (and once more after the final
    join pass) with every link and review item made so far (progress reporting; the lists are not copied).
    Gemini's answers are never cached: every call asks again for every ambiguous pair.
    Returns {"links", "review_items", "questions_asked", "questions_total"}."""
    max_gap = MAX_GAP_REPORTS if max_gap is None else max(0, max_gap)
    by_partition: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
    for n in nodes:
        by_partition[n["partition"]].append(n)
    all_reports = sorted({n["report"] for n in nodes}, key=lambda r: (sk.report_end_year(r), r))

    links: List[Dict[str, Any]] = []
    review: List[Dict[str, Any]] = []
    asked = 0
    total_questions = 0

    states = {}
    for pkey, pnodes in by_partition.items():
        ov = _Overrides(overrides, pkey)
        reports = sorted({n["report"] for n in pnodes}, key=lambda r: (sk.report_end_year(r), r))
        by_report = defaultdict(list)
        for n in sorted(pnodes, key=lambda x: (x["label"], x["section"])):
            if not ov.is_detached(n):
                by_report[n["report"]].append(n)
        label_reports: Dict[str, Set[str]] = defaultdict(set)
        for n in pnodes:
            label_reports[n["label"]].add(n["report"])

        def coexist(a, b, _lr=label_reports):
            return bool(_lr[a] & _lr[b])

        states[pkey] = {"ov": ov, "reports": reports, "by_report": by_report, "coexist": coexist,
                        "heads": {}, "matched_to": set()}

    # Reports are counted in COMPANY order (a report this partition lacks still counts as a step).
    rank = {r: i for i, r in enumerate(all_reports)}

    # Report transitions are processed in company order so Gemini questions are batched per report.
    for k, report in enumerate(all_reports):
        pending: List[Tuple[str, Dict[str, Any]]] = []   # (pkey, question)
        for pkey, st in states.items():
            if report not in st["by_report"]:
                continue
            new_nodes = st["by_report"][report]
            # Nearest first: only lines whose series ended in the immediately previous report.
            heads = [h for h in st["heads"].values() if rank[h["report"]] == k - 1]
            res = _link_pair(pkey, st, heads, new_nodes)
            links.extend(res["links"])
            review.extend(res["review"])
            matched_new = {l["to_node"] for l in res["links"]}
            matched_heads = {l["from_node"] for l in res["links"]}
            pending.extend((pkey, q) for q in res["questions"])
            st["_round"] = {"heads": heads, "new": new_nodes, "k": k,
                            "matched_new": matched_new, "matched_heads": matched_heads, "questions": res["questions"]}

        # R3: one Gemini call per report transition
        questions = [q for _, q in pending]
        total_questions += len(questions)
        resolved, n_asked = _resolve_questions(questions, judge, on_questions, report)
        asked += n_asked

        for pkey, st in states.items():
            rnd = st.pop("_round", None)
            if rnd is None:  # partition has no nodes in this report
                continue
            accepted = _accept_answers(rnd, resolved)
            links.extend(accepted["links"])
            review.extend(accepted["review"])
            rnd["matched_new"] |= {l["to_node"] for l in accepted["links"]}
            rnd["matched_heads"] |= {l["from_node"] for l in accepted["links"]}

            # R4: widen step by step for lines still unmatched: 1 report skipped, then 2, ... up to
            # MAX_GAP_REPORTS. A closer report always wins: a line continued at a smaller gap is gone.
            for gap in range(1, max_gap + 1):
                gap_heads = [h for h in st["heads"].values()
                             if rank[h["report"]] == k - 1 - gap and h["id"] not in rnd["matched_heads"]]
                new_left = [n for n in rnd["new"] if n["id"] not in rnd["matched_new"]]
                if not gap_heads or not new_left:
                    continue
                gl, gr = _widen_link(st, gap_heads, new_left, gap)
                links.extend(gl)
                review.extend(gr)
                rnd["matched_new"] |= {l["to_node"] for l in gl}
                rnd["matched_heads"] |= {l["from_node"] for l in gl}

            # Advance heads: linked heads move to the new node; unmatched new nodes start series
            node_by_id = {n["id"]: n for n in rnd["new"]}
            for l in [l for l in links if l["to_report"] == report and l["partition"] == pkey]:
                st["heads"].pop(l["from_node"], None)
                st["heads"][l["to_node"]] = node_by_id[l["to_node"]]
            for n in rnd["new"]:
                if n["id"] not in rnd["matched_new"]:
                    st["heads"][n["id"]] = n
        if on_step is not None:
            on_step(k + 1, len(all_reports), links, review)

    # R5: lines whose series stopped and lines that start later, joined by one batched Gemini pass
    joined = _join_series(states, links, rank, judge, on_questions)
    links.extend(joined["links"])
    review.extend(joined["review"])
    asked += joined["asked"]
    total_questions += joined["total"]
    if on_step is not None and all_reports:
        on_step(len(all_reports), len(all_reports), links, review)

    return {"links": links, "review_items": review,
            "questions_asked": asked, "questions_total": total_questions}


_QID = itertools.count(1)


def _link_pair(pkey: str, st: Dict[str, Any], heads: List[Dict[str, Any]], new_nodes: List[Dict[str, Any]]) -> Dict[str, Any]:
    """R0, R1, R2, R2s for one adjacent report pair of one partition; collects R3 questions."""
    ov, coexist = st["ov"], st["coexist"]
    links, review, questions = [], [], []
    m_heads: Set[str] = set()
    m_new: Set[str] = set()
    head_by = {h["id"]: h for h in heads}
    new_by = {n["id"]: n for n in new_nodes}

    # R0: user edge links
    for e in ov.force_link_edges:
        hs = [h for h in heads if h["label"] == e["a"] and h["report"] == e["a_report"]]
        ns = [n for n in new_nodes if n["label"] == e["b"] and n["report"] == e["b_report"]]
        for h in hs[:1]:
            for n in ns[:1]:
                if h["id"] not in m_heads and n["id"] not in m_new:
                    links.append(_link(h, n, "USER", "user", 1.0, evidence(h, n), override_id=e.get("id")))
                    m_heads.add(h["id"]); m_new.add(n["id"])

    # R1: same label (or same printed label, "X total", or a user alias); same section preferred.
    # Two passes, so an exact same-label pair is never taken away by a looser match of another line.
    for exact_only in (True, False):
        for h in heads:
            if h["id"] in m_heads:
                continue
            cands = []
            for n in new_nodes:
                if n["id"] in m_new or not _name_match(h, n, ov) or (exact_only and n["label"] != h["label"]):
                    continue
                user_alias = h["label"] != n["label"] and ov.alias_equal(h["label"], n["label"])
                if not user_alias and (h["unit"] != n["unit"] or sk.sections_contrast(h["section"], n["section"])
                                       or _side_clash(h, n)
                                       or (h["label"] != n["label"] and veto(h, n, coexist, ov))):
                    continue
                cands.append(n)
            if not cands:
                continue
            cands.sort(key=lambda n: (n["section"] != h["section"], n.get("cat") != h.get("cat"), n["label"] != h["label"], n["id"]))
            n = cands[0]
            user_alias = h["label"] != n["label"] and ov.alias_equal(h["label"], n["label"])
            rule = "ALIAS_EQUIV" if user_alias else "NAME_EXACT"
            ev = evidence(h, n)
            doubt = None if user_alias else _name_link_doubt(h, n, ev)
            if doubt and doubt.startswith("values differ"):
                better = _value_match_instead_of_name(h, n, heads, new_nodes, m_new, st)
                if better is not None:
                    links.append(_link(h, better, "VALUE_UNIQUE", "auto", 0.95, evidence(h, better),
                                       preferred_over_name=n["label"]))
                    m_heads.add(h["id"]); m_new.add(better["id"])
                    review.append({"kind": "VALUE_CONFLICT", "dedupe_key": f"{h['id']}>{n['id']}",
                                   "payload": {**_pair_payload(h, n), "evidence": ev,
                                               "overlap_sources": _overlap_sources(h, n, ev["overlap"]),
                                               "confidence": 0.0, "rule": rule, "rejected": True,
                                               "reason": f"{doubt}; the old values match '{_display(better)}' exactly, so that line was linked instead"}})
                    continue
            conf = NAME_DOUBT_CONFIDENCE if doubt else 1.0
            links.append(_link(h, n, rule, "user" if user_alias else "auto", conf, ev))
            m_heads.add(h["id"]); m_new.add(n["id"])
            if _doubt_needs_review(doubt, ev):
                review.append({"kind": "VALUE_CONFLICT", "dedupe_key": f"{h['id']}>{n['id']}",
                               "payload": {**_pair_payload(h, n), "evidence": ev,
                                           "overlap_sources": _overlap_sources(h, n, ev["overlap"]),
                                           "confidence": conf, "rule": rule, "reason": doubt}})

    # R2: unique comparative-value match both ways
    rem_h = [h for h in heads if h["id"] not in m_heads]
    rem_n = [n for n in new_nodes if n["id"] not in m_new]
    ev_cache: Dict[Tuple[str, str], Dict[str, Any]] = {}
    soft_cache: Dict[Tuple[str, str], Dict[str, Any]] = {}      # blocked only by an entity word: a question for Fintra AI, never an automatic link
    vetoed: Dict[Tuple[str, str], str] = {}
    for h in rem_h:
        for n in rem_n:
            v = veto(h, n, coexist, ov)
            if v:
                vetoed[(h["id"], n["id"])] = v
                if _soft_veto(v):
                    soft_cache[(h["id"], n["id"])] = evidence(h, n)
                continue
            ev_cache[(h["id"], n["id"])] = evidence(h, n)
    agree_h = defaultdict(list)
    agree_n = defaultdict(list)
    for (hid, nid), ev in ev_cache.items():
        if ev["class"] == sk.AGREE:
            agree_h[hid].append(nid)
            agree_n[nid].append(hid)
    r2 = []
    for hid, nids in agree_h.items():
        if len(nids) == 1 and len(agree_n[nids[0]]) == 1:
            h, n = head_by[hid], new_by[nids[0]]
            if _value_link_supported(h, n):
                r2.append((h, n))
    for h, n in sorted(r2, key=lambda p: (p[0]["label"], p[1]["label"])):
        links.append(_link(h, n, "VALUE_UNIQUE", "auto", 0.95, ev_cache[(h["id"], n["id"])]))
        m_heads.add(h["id"]); m_new.add(n["id"])

    # R2s: split / merge detection (no link, review item)
    rem_h = [h for h in heads if h["id"] not in m_heads]
    rem_n = [n for n in new_nodes if n["id"] not in m_new]
    excluded: Set[str] = set()
    for item in _detect_splits(rem_h, rem_n) + _detect_merges(rem_h, rem_n):
        review.append(item)
        excluded.update(item["payload"]["node_ids"])

    # R3: questions for Gemini
    for h in rem_h:
        if h["id"] in excluded:
            continue
        cands = []
        for n in rem_n:
            ev = ev_cache.get((h["id"], n["id"])) or soft_cache.get((h["id"], n["id"]))
            if n["id"] in excluded or ev is None:
                continue
            support = _lexical_support(h, n)
            cand_pair = are_candidate_pair(_display(h), _display(n))[0]
            if ev["class"] == sk.AGREE:
                ok = True
            elif ev["class"] == sk.NONE:
                ok = cand_pair or (h["section"] == n["section"] != "OTHER" and support >= 0.5)
            else:
                ok = cand_pair
            if ok:
                cands.append((_score(h, n, ev), n, ev))
        if not cands:
            continue
        cands.sort(key=lambda x: (-x[0], x[1]["label"]))
        cands = cands[:MAX_CANDIDATES]
        # "label" is what Gemini sees.
        questions.append({
            "qid": next(_QID), "partition": pkey, "statement_type": h["statement_type"],
            "head_node": h, "head_key": h["label"],
            "head": {"label": _shown(h), "report": h["report"], "section": h["section"]},
            "candidates": [{"label": _shown(n), "key": n["label"], "report": n["report"], "section": n["section"],
                            "evidence": evidence_token(ev), "class": ev["class"], "node": n, "ev": ev}
                           for _, n, ev in cands],
            "unique_agree": len([c for c in cands if c[2]["class"] == sk.AGREE]) == 1
                            and len(agree_h.get(h["id"], [])) == 1,
        })
    return {"links": links, "review": review, "questions": questions}


def _accept_answers(rnd: Dict[str, Any], resolved: Dict[int, Dict[str, Any]]) -> Dict[str, Any]:
    links, review = [], []
    used_heads = set(rnd["matched_heads"])
    used_new = set(rnd["matched_new"])
    answered = []
    for q in rnd["questions"]:
        a = resolved.get(q["qid"])
        if a is None:
            continue
        conf = a.get("confidence", 0.0)
        verdict = a.get("verdict") or ("SAME" if conf >= 0.9 else "UNSURE" if conf > 0 else "DIFFERENT")
        answered.append((conf, verdict, q, a))
    # SAME answers claim a candidate first, then UNSURE ones (a stable order: verdict, then name)
    for conf, verdict, q, a in sorted(answered, key=lambda x: (-x[0], x[2]["head_key"])):
        h = q["head_node"]
        if verdict == "NO_ANSWER":
            # the model left this question out: show it to the user, it is not a "no"
            c0 = q["candidates"][0]
            n0 = c0["node"]
            review.append({"kind": "AMBIGUOUS_LINK", "dedupe_key": f"{h['id']}>{n0['id']}",
                           "payload": {**_pair_payload(h, n0), "evidence": c0["ev"], "confidence": 0.0,
                                       "verdict": "NO_ANSWER", "candidates": [c["label"] for c in q["candidates"]]}})
            continue
        if a["choice"] < 0 or a["choice"] >= len(q["candidates"]) or verdict == "DIFFERENT":
            continue
        c = q["candidates"][a["choice"]]
        n, ev = c["node"], c["ev"]
        # Gemini account types only block a link between balance-sheet kinds (an asset is never a
        # liability or equity line); for profit, cash-flow and other lines its typing is too loose to veto
        types_ok = not (a.get("type_head") in BALANCE_TYPES and a.get("type_choice") in BALANCE_TYPES
                        and a["type_head"] != a["type_choice"])
        payload = {**_pair_payload(h, n), "evidence": ev, "overlap_sources": _overlap_sources(h, n, ev["overlap"]),
                   "confidence": conf, "verdict": verdict}
        if h["id"] in used_heads or n["id"] in used_new:
            continue
        worst, worst_year, flip = value_change(ev, h["statement_type"]) if ev["class"] == sk.DISAGREE else (0.0, None, None)
        hard = worst > NAME_DOUBT_CHANGE or flip is not None
        if verdict == "SAME" and types_ok:
            links.append(_link(h, n, "GEMINI", "gemini", conf, ev))
            used_heads.add(h["id"]); used_new.add(n["id"])
            if ev["class"] == sk.DISAGREE and (flip is not None or high_variance(ev)):
                # Gemini linked them and the figures differ by more than 100 %, or flip sign: shown for review. A smaller
                # difference is a restatement of the cell (it has its own type, RESTATED_LOW / MID) and needs no review item.
                review.append({"kind": "VALUE_CONFLICT", "dedupe_key": f"{h['id']}>{n['id']}", "payload": payload})
        elif ev["class"] == sk.DISAGREE and hard:
            review.append({"kind": "VALUE_CONFLICT", "dedupe_key": f"{h['id']}>{n['id']}", "payload": {**payload, "rejected": True}})
        else:
            review.append({"kind": "AMBIGUOUS_LINK", "dedupe_key": f"{h['id']}>{n['id']}", "payload": payload})
    return {"links": links, "review": review}


def _pair_payload(h: Dict[str, Any], n: Dict[str, Any]) -> Dict[str, Any]:
    return {"partition": h["partition"], "statement_type": h["statement_type"], "scope": h["scope"],
            "entity_key": h["entity_key"], "segment_key": h["segment_key"],
            "a_label_key": h["label"], "a_report": h["report"], "a_field": _display(h),
            "b_label_key": n["label"], "b_report": n["report"], "b_field": _display(n),
            "a_node": h["id"], "b_node": n["id"]}


def _widen_link(st: Dict[str, Any], heads: List[Dict[str, Any]], new: List[Dict[str, Any]],
                gap: int) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
    """R4 at one gap: unmatched new nodes vs series that ended `gap` reports before the previous one.
    gap 1: a name match or a unique value match (GAP_NAME / GAP_VALUE).
    gap 2+: a name match only, linked at lower confidence and flagged for review (LONG_GAP)."""
    ov, coexist = st["ov"], st["coexist"]
    links, review = [], []
    used_h: Set[str] = set()
    used_n: Set[str] = set()

    def payload(h, n, **extra):
        return {"partition": h["partition"], "statement_type": h["statement_type"],
                "scope": h["scope"], "entity_key": h["entity_key"], "segment_key": h["segment_key"],
                "a_label_key": h["label"], "a_report": h["report"], "a_field": _display(h),
                "b_label_key": n["label"], "b_report": n["report"], "b_field": _display(n),
                "a_node": h["id"], "b_node": n["id"], **extra}

    for n in new:
        # R0 across the gap: a user's accepted link between these two reports
        forced = next((h for e in ov.force_link_edges for h in heads
                       if h["id"] not in used_h and h["label"] == e["a"] and h["report"] == e["a_report"]
                       and n["label"] == e["b"] and n["report"] == e["b_report"]), None)
        if forced is not None:
            links.append(_link(forced, n, "USER", "user", 1.0, evidence(forced, n), gap=gap))
            used_h.add(forced["id"]); used_n.add(n["id"])
            continue
        cands = [h for h in heads if h["id"] not in used_h and _name_match(h, n, ov) and h["unit"] == n["unit"]
                 and not sk.sections_contrast(h["section"], n["section"]) and not _side_clash(h, n)]
        if not cands:
            continue
        ok, blocked = [], []
        for h in cands:
            user_alias = h["label"] != n["label"] and ov.alias_equal(h["label"], n["label"])
            reason = None if user_alias else veto(h, n, coexist, ov)
            if not reason and not user_alias and not _heading_compatible(h, n):
                reason = f"headings share no word: '{h.get('cat')}' vs '{n.get('cat')}'"
            (blocked if reason else ok).append((h, reason))
        rank_key = lambda x: (x["section"] == n["section"], x.get("cat") == n.get("cat"), x["label"] == n["label"], x["id"])
        if not ok:
            h, reason = max(blocked, key=lambda x: rank_key(x[0]))
            review.append({"kind": "GAP_BLOCKED", "dedupe_key": f"{h['id']}>{n['id']}",
                           "payload": payload(h, n, reports_skipped=gap, reason=reason)})
            continue
        h = max((x[0] for x in ok), key=rank_key)
        ev = evidence(h, n)
        mag = _magnitude_ratio(h, n)
        jump = {"magnitude_ratio": round(mag[0], 1)} if mag and mag[0] > MAGNITUDE_JUMP_RATIO else {}
        if gap == 1:
            links.append(_link(h, n, "GAP_NAME", "auto", 0.9, ev, gap=gap, **jump))
        else:
            links.append(_link(h, n, "LONG_GAP", "auto", LONG_GAP_CONFIDENCE, ev, gap=gap, **jump))
            review.append({"kind": "LONG_GAP", "dedupe_key": f"{h['id']}>{n['id']}",
                           "payload": payload(h, n, reports_skipped=gap)})
        if jump:
            review.append({"kind": "MAGNITUDE_JUMP", "dedupe_key": f"{h['id']}>{n['id']}",
                           "payload": payload(h, n, reports_skipped=gap, ratio=jump["magnitude_ratio"],
                                              a_year=mag[1], b_year=mag[2],
                                              a_value=h["values"][mag[1]][0], b_value=n["values"][mag[2]][0])})
        used_h.add(h["id"]); used_n.add(n["id"])
    if gap > 1:
        return links, review
    # gap 1 only: by unique value
    agree_n = defaultdict(list)
    agree_h = defaultdict(list)
    evs = {}
    for n in new:
        if n["id"] in used_n:
            continue
        for h in heads:
            if h["id"] in used_h or veto(h, n, coexist, ov):
                continue
            ev = evidence(h, n)
            if ev["class"] == sk.AGREE:
                agree_n[n["id"]].append(h)
                agree_h[h["id"]].append(n)
                evs[(h["id"], n["id"])] = ev
    for n in new:
        hs = agree_n.get(n["id"], [])
        if len(hs) == 1 and len(agree_h[hs[0]["id"]]) == 1:
            h = hs[0]
            if _value_link_supported(h, n):
                links.append(_link(h, n, "GAP_VALUE", "auto", 0.9, evs[(h["id"], n["id"])], gap=gap))
                used_h.add(h["id"]); used_n.add(n["id"])
    return links, review


SUBSET_POOL = 12      # candidate parts tried per line (the most similar names first)


def _line_overlap_year(target: Dict[str, Any], pool: List[Dict[str, Any]]) -> Optional[int]:
    """The latest year that this line and at least one candidate part both print (per line, not per partition)."""
    common = set(target["values"]) & set().union(*(set(p["values"]) for p in pool)) if pool else set()
    return max(common) if common else None


def _subset_matches(target: Dict[str, Any], pool: List[Dict[str, Any]], year: int) -> Optional[List[Dict[str, Any]]]:
    if year not in target["values"]:
        return None
    tv, tm = target["values"][year]
    if sk.informative_weight(tv, target["kind"], tm) < 1.0:
        return None
    # parts must carry a real value: a zero (or tiny round) line adds nothing to a sum, so "A = A + 0"
    # would be reported as a split and the zero line (e.g. an unused "Money at call") never gets linked
    cands = [p for p in pool if year in p["values"] and p["kind"] == target["kind"] and p["unit"] == target["unit"]
             and sk.informative_weight(p["values"][year][0], p["kind"], p["values"][year][1]) >= 1.0
             and (p["section"] == target["section"] or "OTHER" in (p["section"], target["section"]))]
    # parts usually share words with the line they come from: try the most similar names first
    cands.sort(key=lambda p: (-_lexical_support(target, p), p["id"]))
    cands = cands[:SUBSET_POOL]
    for size in (2, 3):
        for combo in itertools.combinations(cands, size):
            total = sum(p["values"][year][0] for p in combo)
            if sk.agree(tv, total, target["kind"], tm, max(p["values"][year][1] for p in combo)):
                return list(combo)
    return None


def _detect_splits(heads: List[Dict[str, Any]], new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    out = []
    used: Set[str] = set()
    for h in heads:
        pool = [n for n in new if n["id"] not in used]
        year = _line_overlap_year(h, pool)
        parts = _subset_matches(h, pool, year) if year is not None else None
        if parts:
            used.update(p["id"] for p in parts)
            out.append({"kind": "SPLIT_DETECTED", "dedupe_key": f"{h['id']}>" + "+".join(p["id"] for p in parts),
                        "payload": {"partition": h["partition"], "statement_type": h["statement_type"],
                                    "scope": h["scope"], "year": year, "from": {"report": h["report"], "field": _display(h),
                                    "label_key": h["label"], "value": h["values"][year][0]},
                                    "into": [{"report": p["report"], "field": _display(p), "label_key": p["label"],
                                              "value": p["values"][year][0]} for p in parts],
                                    "node_ids": [h["id"]] + [p["id"] for p in parts]}})
    return out


def _detect_merges(heads: List[Dict[str, Any]], new: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    out = []
    used: Set[str] = set()
    for n in new:
        pool = [h for h in heads if h["id"] not in used]
        year = _line_overlap_year(n, pool)
        parts = _subset_matches(n, pool, year) if year is not None else None
        if parts:
            used.update(p["id"] for p in parts)
            out.append({"kind": "MERGE_DETECTED", "dedupe_key": "+".join(p["id"] for p in parts) + f">{n['id']}",
                        "payload": {"partition": n["partition"], "statement_type": n["statement_type"],
                                    "scope": n["scope"], "year": year, "into": {"report": n["report"], "field": _display(n),
                                    "label_key": n["label"], "value": n["values"][year][0]},
                                    "from": [{"report": p["report"], "field": _display(p), "label_key": p["label"],
                                              "value": p["values"][year][0]} for p in parts],
                                    "node_ids": [n["id"]] + [p["id"] for p in parts]}})
    return out


# ---------------------------------------------------------------------------
# Series
# ---------------------------------------------------------------------------
def build_series(nodes: List[Dict[str, Any]], links: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]:
    """Chains linked nodes into series; breaks chains whose labels are printed side by side somewhere.

    Returns (series, kept_links, review_items). Each series: {"partition", "nodes": [node...] (report order), "links": [...]}."""
    node_by = {(n["partition"], n["id"]): n for n in nodes}
    label_reports: Dict[Tuple[str, str], Set[str]] = defaultdict(set)
    for n in nodes:
        label_reports[(n["partition"], n["label"])].add(n["report"])
    review = []
    active = list(links)
    while True:
        nxt = {(l["partition"], l["from_node"]): l for l in active}
        has_prev = {(l["partition"], l["to_node"]) for l in active}
        chains = []
        for key, n in node_by.items():
            if key in has_prev:
                continue
            chain_nodes, chain_links = [n], []
            cur = key
            while cur in nxt:
                l = nxt[cur]
                chain_links.append(l)
                cur = (l["partition"], l["to_node"])
                chain_nodes.append(node_by[cur])
            chains.append((chain_nodes, chain_links))
        broken = False
        for chain_nodes, chain_links in chains:
            labels = []
            for n in chain_nodes:
                if n["label"] not in labels:
                    labels.append(n["label"])
            pkey = chain_nodes[0]["partition"]
            bad = None
            for a, b in itertools.combinations(labels, 2):
                # two names of one line are never printed together while the chain still used the earlier name;
                # an earlier name that comes back LATER as a new line (a redefinition) is not a contradiction
                last_a = max(sk.report_end_year(n["report"]) for n in chain_nodes if n["label"] == a)
                if any(sk.report_end_year(r) <= last_a for r in label_reports[(pkey, a)] & label_reports[(pkey, b)]):
                    bad = (a, b)
                    break
            if not bad:
                continue
            idx = [i for i, n in enumerate(chain_nodes) if n["label"] in bad]
            between = chain_links[min(idx):max(idx)]
            breakable = [l for l in between if l["rule"] not in ("USER", "ALIAS_EQUIV")] or between
            weakest = min(breakable, key=lambda l: (l["confidence"], l["to_report"]))
            active.remove(weakest)
            review.append({"kind": "SERIES_INCONSISTENT",
                           "dedupe_key": f"{weakest['from_node']}>{weakest['to_node']}",
                           "payload": {"partition": pkey, "labels": list(bad), "broken_link": {
                               k: weakest[k] for k in ("from_report", "to_report", "from_label", "to_label", "rule")}}})
            broken = True
            break
        if not broken:
            break
    series = [{"partition": c[0][0]["partition"], "nodes": c[0], "links": c[1]} for c in chains]
    series.sort(key=lambda s: (s["partition"], s["nodes"][-1]["label"], s["nodes"][0]["report"]))
    return series, active, review


def resolve_series_values(series: Dict[str, Any]) -> Dict[str, Any]:
    """Values per year (latest report wins), stating years, restatement and micro variances."""
    nodes = series["nodes"]
    kind = nodes[-1]["kind"]
    obs = defaultdict(list)
    for n in nodes:
        for y, (v, m) in n["values"].items():
            obs[y].append((n["report_end"], n["report"], v, m, n, bool(n.get("_restated", {}).get(y))))
            for pv, pm, pprov in n.get("prints", {}).get(y, ()):
                # the report's own print of a year it also restates: an earlier step of that year's history
                obs[y].append((n["report_end"], n["report"], pv, pm, {**n, "prov": {**n["prov"], y: pprov}}, False))
    values, stating, prov, restate, micro, alt = {}, {}, {}, {}, {}, {}
    for y in sorted(obs):
        lst = sorted(obs[y], key=lambda o: (o[0], o[1], o[5]))
        latest, earliest = lst[-1], lst[0]
        values[y] = latest[2]
        stating[y] = latest[0]
        prov[y] = latest[4]["prov"][y]
        # every consecutive pair of reports that print this year is compared, so a figure that changes in a middle
        # report and goes back is still a restatement (comparing only the first and the last report would miss it)
        steps = []
        for old, new in zip(lst, lst[1:]):
            if old[2] != new[2]:
                steps.append({"from_report": old[1], "to_report": new[1], "from_value": old[2], "to_value": new[2],
                              "difference": new[2] - old[2],
                              "percent_change": round((new[2] - old[2]) / old[2] * 100, 2) if old[2] else None,   # None = zero base
                              "within_tolerance": sk.agree(old[2], new[2], kind, old[3], new[3])})
        if steps:
            diff = latest[2] - earliest[2]
            pct = round(diff / earliest[2] * 100, 2) if earliest[2] else None            # None = n/a (zero base)
            if all(st["within_tolerance"] for st in steps):
                if earliest[2] == latest[2]:
                    continue            # drifted by rounding and came back: nothing to show
                big = max(abs(earliest[2]), abs(latest[2]))
                micro[str(y)] = {"year": str(y), "field_a": _display(earliest[4]), "field_b": _display(latest[4]),
                                 "val_a": earliest[2], "val_b": latest[2], "diff": abs(diff),
                                 **source_cell(earliest[1], earliest[4]["prov"].get(y), "a"),
                                 **source_cell(latest[1], latest[4]["prov"].get(y), "b"),
                                 "percent_change": round(abs(diff) / big * 100, 6) if big else 0.0,
                                 "type": "MICRO_TOLERANCE_VARIANCE", "canonical_val": latest[2],
                                 "reason": f"Rounding difference between the {earliest[1]} and {latest[1]} reports for {y}"}
            else:
                restate[str(y)] = {"original_value": earliest[2], "restated_value": latest[2],
                                   "difference": diff, "percent_change": pct, "history": steps}
    for n in nodes:
        for y, lst in n["alt_periods"].items():
            alt.setdefault(str(y), []).extend({"months": a["months"], "value": a["value"], "report_year": a["report_year"],
                                               "page_number": a["page_number"]} for a in lst)
    return {"values": values, "stating": stating, "prov": prov, "restate": restate, "micro": micro, "alt": alt}


def _scope_sort(scope: str) -> int:
    return 0 if scope == sk.CONSOLIDATED else 1


SCOPE_LABELS = {sk.CONSOLIDATED: "Consolidated", sk.STANDALONE: "Standalone"}


def display_name(base: str, scope: str, entity: Optional[str], segment: Optional[str] = None) -> str:
    """The line name shown to users: the plain name for both scopes (the results are grouped by scope, so a consolidated line does
    not carry "Consolidated"), " - <entity>" appended when the line belongs to an entity other than the reporting company
    ("Total assets - City Brokerage Limited"), and " - <segment>" for a segment's line ("Segment liabilities - Conventional")."""
    name = base
    if entity:
        name += f" - {str(entity).strip()}"
    if segment and str(segment).strip().lower() not in base.lower():
        name += f" - {str(segment).strip()}"
    return name


def _part_ident(c: Dict[str, Any]) -> Tuple:
    return ("s", c["series_id"]) if c.get("series_id") is not None else ("f", c.get("field"))


def _union_parts(lists: List[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
    """The parts of several printings of one total as one ordered list. The first list (the latest report) keeps its order and
    signs; a part only an older list has is inserted right after the nearest part before it in its own list that is already in
    the result, else right before the nearest one after it, else appended."""
    out: List[Dict[str, Any]] = []
    idents: List[Tuple] = []
    for parts in lists:
        for i, c in enumerate(parts):
            k = _part_ident(c)
            if k in idents:
                continue
            pos = None
            for j in range(i - 1, -1, -1):
                if _part_ident(parts[j]) in idents:
                    pos = idents.index(_part_ident(parts[j])) + 1
                    break
            if pos is None:
                for j in range(i + 1, len(parts)):
                    if _part_ident(parts[j]) in idents:
                        pos = idents.index(_part_ident(parts[j]))
                        break
            if pos is None:
                pos = len(out)
            out.insert(pos, dict(c))
            idents.insert(pos, k)
    return out


def year_breakdowns(series_list: List[Dict[str, Any]], series_ids: List[int],
                    compositions: List[Dict[str, Any]]) -> Dict[Tuple, Dict[str, Any]]:
    """The breakdown each report PRINTED for each total cell, lifted to series: {(report, page, table, row, col): {"report", "status",
    "components": [{"series_id", "sign", "value" (as printed)} (+ "field" for a part whose cell is in no series)]}}. Only SUM compositions whose
    total cell belongs to a series. A composition that does not add up to its own printed total (status MISMATCH) is listed with its
    status only, no components: it exists but is no basis for a year. A total's `breakdown.by_year[year]` is the entry of the cell the
    year's value comes from, so the Standard layout can add up a year from the very report that states it (not from the union of every
    report's parts), and knows that a year with no entry has no stored breakdown at all (nothing to check)."""
    cell_sid: Dict[Tuple, int] = {}
    for s, sid in zip(series_list, series_ids):
        for n in s["nodes"]:
            for (pg, tn, r, c) in n.get("cells") or ():
                cell_sid[(n["report"], pg, tn, r, c)] = sid
    out: Dict[Tuple, Dict[str, Any]] = {}
    for comp in compositions:
        if (comp.get("operation") or "SUM") != "SUM":
            continue
        fy, pg, tn = comp["fiscal_year"], comp["page_number"], comp["table_num"]
        t = comp["total"]
        t_sid = cell_sid.get((fy, pg, tn, t["row"], t["col"]))
        if t_sid is None:
            continue
        if comp.get("status") != "MATCH":
            out[(fy, pg, tn, t["row"], t["col"])] = {"report": fy, "status": comp.get("status")}
            continue
        parts = []
        for p in comp["parts"]:
            p_sid = cell_sid.get((fy, pg, tn, p["row"], p["col"]))
            if p_sid == t_sid:
                continue
            # `value` is the figure as printed: used when the part's series is not on the sheet of the total, or is in no series
            part: Dict[str, Any] = {"series_id": p_sid, "sign": p["sign"], "value": p.get("value")}
            if p_sid is None:
                part["field"] = p.get("field") or p.get("raw_field")
            parts.append(part)
        if parts:
            out[(fy, pg, tn, t["row"], t["col"])] = {"report": fy, "status": comp["status"], "components": parts}
    return out


def series_breakdowns(series_list: List[Dict[str, Any]], series_ids: List[int],
                      compositions: List[Dict[str, Any]]) -> Tuple[Dict[int, Dict[str, Any]], Dict[int, List[int]]]:
    """Cell compositions (database.get_cell_compositions) lifted to series: for each total series, its components as the UNION
    of the parts of every printing of the total in every report (the latest report's order and signs first, a line only an older
    report printed at its printed place; `report`, `direction`, `operation` and `status` are the latest report's, and each
    component has `last_report`, the latest report that printed it); and for each series, the totals it is part of.
    Returns ({series_id: {"report", "direction", "status", "components": [{"series_id", "field", "sign", "last_report"}]}},
             {series_id: [total series ids]})."""
    cell_sid: Dict[Tuple, int] = {}
    sid_name = {sid: _display(s["nodes"][-1]) for s, sid in zip(series_list, series_ids) if s["nodes"]}
    for s, sid in zip(series_list, series_ids):
        for n in s["nodes"]:
            for (pg, tn, r, c) in n.get("cells") or ():
                cell_sid[(n["report"], pg, tn, r, c)] = sid
    printings: Dict[int, List[Tuple[Tuple, Dict[str, Any], List[Dict[str, Any]]]]] = defaultdict(list)
    part_of: Dict[int, Set[int]] = defaultdict(set)
    for comp in compositions:
        fy, pg, tn = comp["fiscal_year"], comp["page_number"], comp["table_num"]
        t_sid = cell_sid.get((fy, pg, tn, comp["total"]["row"], comp["total"]["col"]))
        if t_sid is None:
            continue
        components = []
        for p in comp["parts"]:
            p_sid = cell_sid.get((fy, pg, tn, p["row"], p["col"]))
            if p_sid == t_sid:
                continue
            components.append({"series_id": p_sid, "field": sid_name.get(p_sid) or p.get("field") or p.get("raw_field"),
                               "sign": p["sign"]})
            if p_sid is not None:
                part_of[p_sid].add(t_sid)
        if not components:
            continue
        printings[t_sid].append(((sk.report_end_year(fy), fy), comp, components))
    per_total: Dict[int, Dict[str, Any]] = {}
    for t_sid, items in printings.items():
        items.sort(key=lambda it: it[0], reverse=True)           # newest report first; ties keep their order
        last_report: Dict[Tuple, str] = {}
        for _, comp, parts in items:
            for c in parts:
                last_report.setdefault(_part_ident(c), comp["fiscal_year"])
        head = items[0][1]
        per_total[t_sid] = {"report": head["fiscal_year"], "direction": head["direction"],
                            "operation": head.get("operation", "SUM"), "status": head["status"],
                            "components": [{**c, "last_report": last_report[_part_ident(c)]}
                                           for c in _union_parts([parts for _, _, parts in items])]}
    return per_total, {k: sorted(v) for k, v in part_of.items()}


def series_to_rows(series_list: List[Dict[str, Any]], series_ids: List[int],
                   overrides: List[Dict[str, Any]], conflicts: List[Dict[str, Any]],
                   compositions: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:
    """The merge result in the shape /results/merged and the merged CSV expect (plus additive series fields)."""
    renames = {}
    for o in sorted(overrides, key=lambda x: (x.get("created_at") or "", x.get("id") or 0)):
        if o["kind"] == "RENAME" and o.get("new_label"):
            renames[(_override_partition(o), sk.label_key(o.get("a_label_key") or ""))] = o["new_label"]

    conflicts_by_field = defaultdict(list)
    for c in conflicts:
        conflicts_by_field[(c["statement_type"], c["field_a"])].append(c)
        conflicts_by_field[(c["statement_type"], c["field_b"])].append(c)

    rows, report_items = [], []
    years = sorted({y for s in series_list for n in s["nodes"] for y in n["values"]})
    breakdowns, breakdown_of = series_breakdowns(series_list, series_ids, compositions or [])
    printed_cells = year_breakdowns(series_list, series_ids, compositions or []) if breakdowns else {}
    for s, sid in zip(series_list, series_ids):
        nodes = s["nodes"]
        latest = nodes[-1]
        res = resolve_series_values(s)
        if not res["values"]:
            continue
        canonical = latest["fields"][0] if latest["fields"] else latest["label"]
        for n in reversed(nodes):
            new_name = renames.get((s["partition"], n["label"]))
            if new_name:
                canonical = new_name
                break
        merged_from = [canonical]
        for n in reversed(nodes):
            for f in n["fields"]:
                if f not in merged_from:
                    merged_from.append(f)
        link_into = {l["to_node"]: l for l in s["links"]}
        constituents = []
        for n in nodes:
            l = link_into.get(n["id"])
            first_year = min(n["prov"]) if n["prov"] else None
            p = n["prov"].get(max(n["prov"])) if n["prov"] else {}
            constituents.append({
                "field": _display(n), "scope": n["scope"],
                "values": {str(y): v for y, (v, _) in sorted(n["values"].items())},
                "page_number": p.get("page_number"), "report_year": n["report"],
                "table_num": p.get("table_num"), "table_row": p.get("table_row"), "table_col": p.get("table_col"),
                "raw_field": p.get("raw_field") or _display(n),
                "provenance": {k: p.get(k) for k in ("report_year", "page_number", "table_num", "table_row", "table_col", "raw_field")} if p else None,
                "restatement_dots": {}, "alias_id": n["id"],
                "link_rule": l["rule"] if l else None, "link_confidence": l["confidence"] if l else None,
                "flags": n["flags"], "first_year": first_year,
            })
        sign_years = sorted({str(y) for l in s["links"] for y in l["evidence"].get("sign_convention_changed", [])})
        latest_year = max(res["values"])
        lp = res["prov"][latest_year]
        mm = []
        seen = set()
        for f in merged_from:
            for c in conflicts_by_field.get((latest["statement_type"], f), []):
                k = (c["field_a"], c["field_b"])
                if k not in seen:
                    seen.add(k)
                    mm.append(c)
        values = {str(y): res["values"].get(y) for y in years}
        breakdown = breakdowns.get(sid)
        if breakdown:
            by_year = {}
            for y, pv in res["prov"].items():           # the breakdown printed with the very cell the year's value comes from
                e = printed_cells.get((pv.get("report_year"), pv.get("page_number"), pv.get("table_num"), pv.get("table_row"), pv.get("table_col")))
                if e:
                    by_year[str(y)] = e
            breakdown = {**breakdown, "by_year": by_year}          # always present (maybe empty): the row is in the per-year format
        shown = display_name(canonical, latest["scope"], latest["entity"], latest["segment"])
        row = {
            "field": shown,
            "base_field": canonical,
            "scope_label": SCOPE_LABELS.get(latest["scope"], latest["scope"].title()),
            "series_id": sid,
            "concept_name": latest["concept_name"],
            "concept": {"name": latest["concept_name"], "statement": latest["statement_type"],
                        "category": latest["category"], "level": latest["aggregation_level"]},
            "scope_obj": {"type": latest["scope"], "entity": latest["entity"], "segment": latest["segment"]},
            "behavior": {"temporal": latest["temporal"], "action": latest["action"],
                         "metric_type": latest["kind"], "qualifier": latest["qualifier"]},
            "category": latest["category"], "aggregation_level": latest["aggregation_level"],
            "entity": latest["entity"], "segment": latest["segment"], "temporal_state": latest["temporal"],
            "action": latest["action"], "metric_type": latest["kind"], "qualifier": latest["qualifier"],
            "unit": latest["unit"], "tag": latest["tag"], "statement_type": latest["statement_type"],
            "scope": latest["scope"],
            "values": values,
            "stating_years": {str(y): v for y, v in res["stating"].items()},
            "restated_variance_years": sorted(res["restate"]),
            "restatement_variances": res["restate"],
            "has_micro_variance": bool(res["micro"]), "micro_variances": res["micro"],
            "micro_variance_years": sorted(res["micro"]),
            "has_value_mismatch": bool(mm), "value_mismatches": mm,
            "mismatch_years": sorted({str(y) for c in mm for y in c.get("conflicting_years", [])}),
            "merged_from": merged_from,
            "constituent_details": constituents,
            "page_number": lp.get("page_number"), "report_year": lp.get("report_year"),
            "table_num": lp.get("table_num"), "table_row": lp.get("table_row"), "table_col": lp.get("table_col"),
            "raw_field": lp.get("raw_field"),
            "provenance": {k: lp.get(k) for k in ("report_year", "page_number", "table_num", "table_row", "table_col", "raw_field")},
            "alias_ids": [n["id"] for n in nodes],
            "link_summary": [{"from_report": l["from_report"], "to_report": l["to_report"], "rule": l["rule"],
                              "confidence": l["confidence"], "gap": l.get("gap", 0)} for l in s["links"]],
            "alt_periods": res["alt"],
            "review_flags": sorted({f for n in nodes for f in n["flags"]} | ({"SIGN_CONVENTION_CHANGED"} if sign_years else set())),
            "sign_convention_years": sign_years,
            "breakdown": breakdown,
            "breakdown_of": breakdown_of.get(sid, []),
        }
        rows.append(row)
        if len(set(merged_from)) > 1:
            confs = [l["confidence"] for l in s["links"]] or [1.0]
            rule_counts = Counter(l["rule"] for l in s["links"])
            report_items.append({
                "canonical_field": shown, "statement_type": latest["statement_type"], "unit": latest["unit"],
                "merged_fields": merged_from, "reporting_years_count": len(res["values"]),
                "confidence": round(min(confs), 4), "series_id": sid,
                "rationale": "Linked across reports by " + ", ".join(f"{n} {r}" for r, n in sorted(rule_counts.items())),
                "has_micro_variance": bool(res["micro"]), "micro_variances": res["micro"],
                "micro_variance_years": sorted(res["micro"]),
            })

    def sort_key(r):
        try:
            s_idx = STATEMENT_TYPE_ORDER.index(r["statement_type"])
        except ValueError:
            s_idx = 999
        return (s_idx, normalize_text(r["base_field"]), _scope_sort(r["scope"]), r["series_id"] or 0)

    rows.sort(key=sort_key)
    return {"rows": rows, "merge_report": report_items, "years": years}


def rows_to_csv(rows: List[Dict[str, Any]], years: List[int]) -> Tuple[str, str, List[str]]:
    import csv
    import io
    headers = ["field", "unit", "tag", "statement_type"] + [str(y) for y in years]
    out, out_st = io.StringIO(), io.StringIO()
    w, w_st = csv.writer(out), csv.writer(out_st)
    w.writerow(headers)
    w_st.writerow(headers)
    for r in rows:
        # the flat CSV has no scope column, so a consolidated line keeps its prefix here
        base = [f"Consolidated {r['field']}" if r.get("scope") == sk.CONSOLIDATED else r["field"], r["unit"], r["tag"], r["statement_type"]]
        w.writerow(csv_safe_row(base + [format_numeric_value(r["values"].get(str(y))) for y in years]))
        w_st.writerow(csv_safe_row(base + [str(r["stating_years"].get(str(y)) or "") for y in years]))
    return out.getvalue(), out_st.getvalue(), headers


def value_conflicts_from_review(review_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """VALUE_MISMATCH conflicts (the merged rows' value_mismatches) for links made although values disagree: Gemini links,
    and same-name links whose change is large or whose headings differ (payload "rule" NAME_EXACT / ALIAS_EQUIV)."""
    out = []
    for item in review_items:
        if item["kind"] != "VALUE_CONFLICT":
            continue
        p = item["payload"]
        diffs = [o for o in p["evidence"]["overlap"] if not o["match"]]
        out.append({
            "type": "VALUE_MISMATCH", "field_a": p["a_field"], "field_b": p["b_field"],
            "statement_type": p["statement_type"], "confidence": round(p.get("confidence") or 0.0, 4),
            "rationale": (("Same name, but the values match another line: not linked" if p.get("rejected")
                           else "Linked by name, but the values differ on common years") if p.get("rule") in ("NAME_EXACT", "ALIAS_EQUIV")
                          else "Fintra AI matched these lines across reports, but their values differ on common years"),
            **({"definition_change": p["definition_change"]} if p.get("definition_change") else {}),
            **({"comparison": p["comparison"]} if p.get("comparison") else {}),
            "conflicting_years": [str(o["year"]) for o in diffs],
            "collision_details": [{"year": str(o["year"]), "field_a": p["a_field"], "field_b": p["b_field"],
                                   "val_a": o["a"], "val_b": o["b"], "diff": abs(o["a"] - o["b"]),
                                   **(p.get("overlap_sources") or {}).get(str(o["year"]), {})} for o in diffs],
            "reason": f"'{p['a_field']}' ({p['a_report']}) and '{p['b_field']}' ({p['b_report']}) "
                      + ("were linked although values differ" if not p.get("rejected") else "were not linked: values differ")
                      + (f" ({p['reason']})" if p.get("rule") and p.get("reason") else ""),
            "linked": not p.get("rejected"),
        })
    return out
