import pytest
from fastapi.testclient import TestClient

from app.main import app
from app.ocr import config as ocr_config
from tests.test_api import OCR_ROUTES

KEY = {"X-API-Key": "k3y", "X-Username": "tester"}

# Concrete values for templated OCR paths.
_SAMPLE = {"{code}": "CITYBANK", "{fy}": "2024-25", "{file_path}": "x.md",
           "{page_number}": "1", "{job_id}": "0" * 32,
           "{series_id}": "1", "{item_id}": "1", "{override_id}": "1"}


def _concrete(path):
    for k, v in _SAMPLE.items():
        path = path.replace(k, v)
    return path


@pytest.fixture
def ocr(monkeypatch):
    """Client with lifespan run (creates OCR dirs + schema in the temp DB)."""
    monkeypatch.setenv("API_KEY", "k3y")
    with TestClient(app, raise_server_exceptions=False) as c:
        yield c


# ---------------------------------------------------------------- auth
PUBLIC_OCR_ROUTES = {("/ocr/health", ("GET",))}


@pytest.mark.parametrize("path,methods", sorted(OCR_ROUTES - PUBLIC_OCR_ROUTES))
def test_every_ocr_route_requires_api_key(ocr, path, methods):
    for method in methods:
        for headers in ({}, {"X-API-Key": "nope"}, {"Authorization": "Bearer k3y"}):
            r = ocr.request(method, _concrete(path), headers=headers)
            assert r.status_code == 401, (method, path, headers, r.status_code)
            assert r.json() == {"detail": "Not authenticated"}


def test_api_key_header_accepted_on_ocr(ocr):
    assert ocr.get("/ocr/company/X/status", headers=KEY).status_code == 200


def test_query_token_not_accepted(ocr, monkeypatch, tmp_path):
    (tmp_path / "pages").mkdir()
    (tmp_path / "pages" / "p.png").write_bytes(b"png")
    monkeypatch.setattr(ocr_config, "STORAGE_DIR", tmp_path)
    assert ocr.get("/ocr/files/pages/p.png", params={"token": "k3y"}).status_code == 401
    assert ocr.get("/ocr/files/pages/p.png", headers=KEY).status_code == 200


# ---------------------------------------------------------------- health
def test_health_is_public_and_minimal(ocr, monkeypatch):
    for name in ("OCR_API_KEY", "EXTRACT_API_KEY", "MERGE_API_KEY"):
        monkeypatch.setenv(name, "x")
    r = ocr.get("/ocr/health")
    assert r.status_code == 200
    assert r.headers["content-type"].startswith("application/health+json")
    assert r.json() == {"status": "pass"}


def test_health_warn_without_gemini_key(ocr, monkeypatch):
    for name in ("OCR_API_KEY", "EXTRACT_API_KEY"):
        monkeypatch.setenv(name, "x")
    monkeypatch.delenv("MERGE_API_KEY", raising=False)              # one step without a key
    r = ocr.get("/ocr/health")
    assert r.status_code == 200 and r.json() == {"status": "warn"}


def test_health_fail_when_db_unreachable(ocr, monkeypatch):
    from app.ocr import database as ocr_database

    def boom():
        raise RuntimeError("db down")

    monkeypatch.setattr(ocr_database, "get_connection", boom)
    r = ocr.get("/ocr/health")
    assert r.status_code == 503 and r.json() == {"status": "fail"}


def test_old_api_prefix_gone(ocr):
    assert ocr.get("/api/health", headers=KEY).status_code == 404


# ---------------------------------------------------------------- info leaks
def test_md_file_listing_has_no_absolute_path(ocr):
    body = ocr.get("/ocr/ocr-md-files", headers=KEY).json()
    assert body["storage_path"] == "storage/ocr-md-files"


# ---------------------------------------------------------------- data
def test_companies_catalog_loaded(ocr):
    companies = ocr.get("/ocr/companies", headers=KEY).json()
    assert len(companies) > 100
    # Other tests may register extra companies in the shared test DB.
    catalog_codes = {c["security_code"].upper() for c in ocr.get("/ocr/companies/catalog", headers=KEY).json()}
    assert catalog_codes <= {c["instrument_code"] for c in companies}


# ---------------------------------------------------------------- validation
@pytest.mark.parametrize("code,fy", [
    ("..", "2024-25"), ("A.B", "2024-25"), ("CITYBANK", "..%2F..%2Fx"),
    ("CITYBANK", "2024-2025"), ("CITY%2FBANK", "2024-25"),
])
def test_path_params_rejected(ocr, code, fy):
    r = ocr.get(f"/ocr/company/{code}/year/{fy}/pages", headers=KEY)
    assert r.status_code in (404, 422)


def test_codes_with_symbols_accepted(ocr):
    for code in ("AMCL(PRAN)", "KAY&QUE"):
        assert ocr.get(f"/ocr/company/{code}/status", headers=KEY).status_code == 200


@pytest.mark.parametrize("body", [
    {"instrument_code": "../x", "fiscal_year": "2024-25", "page_numbers": [1]},
    {"instrument_code": "AAA", "fiscal_year": "../../x", "page_numbers": [1]},
    {"instrument_code": "AAA", "fiscal_year": "2024-25", "page_numbers": [0]},
])
def test_pipeline_body_validated(ocr, body):
    assert ocr.post("/ocr/pipeline/ocr", headers=KEY, json=body).status_code == 422


def _pdf(name="a.pdf", content=b"%PDF-1.4\n%%EOF\n"):
    return ("files", (name, content, "application/pdf"))


def test_upload_rejects_traversal_in_fiscal_year_map(ocr):
    r = ocr.post("/ocr/upload", headers=KEY, data={
        "instrument_code": "AAA", "fiscal_years_json": '{"a.pdf": "../../evil"}'},
        files=[_pdf()])
    assert r.status_code == 422


def test_upload_rejects_bad_json(ocr):
    r = ocr.post("/ocr/upload", headers=KEY, data={
        "instrument_code": "AAA", "fiscal_years_json": "not json"}, files=[_pdf()])
    assert r.status_code == 422


def test_upload_rejects_non_pdf(ocr):
    r = ocr.post("/ocr/upload", headers=KEY, data={
        "instrument_code": "AAA", "fiscal_years_json": "{}"},
        files=[_pdf(content=b"MZ not a pdf")])
    assert r.status_code == 415


def test_upload_size_limit(ocr, monkeypatch):
    monkeypatch.setattr(ocr_config, "MAX_UPLOAD_BYTES", 10)
    r = ocr.post("/ocr/upload", headers=KEY, data={
        "instrument_code": "AAA", "fiscal_years_json": "{}"},
        files=[_pdf(content=b"%PDF-" + b"x" * 100)])
    assert r.status_code == 413


def test_client_pages_reject_bad_page_numbers(ocr):
    png = ("files", ("page_1.png", b"\x89PNG\r\n\x1a\n" + b"0" * 10, "image/png"))
    r = ocr.post("/ocr/pipeline/ocr-client-pages", headers=KEY, data={
        "instrument_code": "AAA", "fiscal_year": "2024-25",
        "page_numbers_json": '["../x"]'}, files=[png])
    assert r.status_code == 422


# ---------------------------------------------------------------- file serving
def test_files_endpoint_serves_inside_storage_only(ocr, monkeypatch, tmp_path):
    storage = tmp_path / "storage"
    (storage / "pages").mkdir(parents=True)
    (storage / "pages" / "p.md").write_text("hello", encoding="utf-8")
    (tmp_path / "secret.txt").write_text("secret", encoding="utf-8")
    monkeypatch.setattr(ocr_config, "STORAGE_DIR", storage)

    r = ocr.get("/ocr/files/pages/p.md", headers=KEY)
    assert r.status_code == 200 and r.text == "hello"
    for bad in ("/ocr/files/..%2Fsecret.txt", "/ocr/files/pages/..%2F..%2Fsecret.txt",
                "/ocr/files/pages", "/ocr/files/missing.md"):
        assert ocr.get(bad, headers=KEY).status_code == 404, bad


# ---------------------------------------------------------------- extraction data (new schema)
_MD = (
    "| Particulars | Note | 2025 | 2024 |\n"
    "|---|---|---|---|\n"
    "| Total assets | 4 | 1,000 | 900 |\n"
)


def _item(field, values, stmt="BALANCE_SHEET", scope="CONSOLIDATED"):
    return {
        "concept": {"name": field, "statement": stmt},
        "scope": {"type": scope},
        "raw_field": field,
        "data": [
            {"year": y, "numeric_value": v, "unit": "BDT", "multiplier": 1.0,
             "table_num": 1, "table_row": 1, "table_col": col}
            for col, (y, v) in enumerate(values, start=3)
        ],
    }


def full_results(client, code):
    """A company's raw and merged results as one dict (the two parts of the results are separate endpoints)."""
    raw = client.get(f"/ocr/company/{code}/results/raw", headers=KEY, params={"view": "full"})
    assert raw.status_code == 200, raw.text
    merged = client.get(f"/ocr/company/{code}/results/merged", headers=KEY, params={"view": "full"})
    assert merged.status_code == 200, merged.text
    out, body = {**raw.json(), **merged.json()}, merged.json()
    while body["has_more"]:                      # the merged table comes one statement at a time
        body = client.get(f"/ocr/company/{code}/results/merged", headers=KEY,
                          params={"statement_type": body["next_statement_type"], "view": "full"}).json()
        for k in ("merged_rows", "value_mismatch_conflicts", "merge_report"):
            out[k] = out[k] + body[k]
    out["value_mismatch_count"] = len(out["value_mismatch_conflicts"])
    return out


def merge_and_find(client, code, field, scope="CONSOLIDATED"):
    """Runs the merge of a company and returns the merged row of `field` (the cell-details lookup is by `series_id`)."""
    assert client.post("/ocr/pipeline/merge", headers=KEY, json={"instrument_code": code}).status_code == 200
    rows = full_results(client, code)["merged_rows"]
    return next(r for r in rows if r["base_field"].lower() == field.lower() and r["scope"] == scope)


def _seed_page(code, fy, page, md, items):
    from app.ocr import database
    database.upsert_page(None, code, fy, page, "", is_selected=True)
    database.update_page_ocr(code, fy, page, md)
    database.save_extracted_metrics(code, fy, page, items)


@pytest.fixture
def seeded(ocr):
    """Two annual reports; the 2023-24 report states 2024 total assets differently (a restatement)."""
    _seed_page("SEEDCO", "2024-25", 5, _MD, [_item("Total assets", [(2025, 1000), (2024, 900)])])
    _seed_page("SEEDCO", "2023-24", 7, _MD, [_item("Total assets", [(2024, 950), (2023, 800)])])
    return ocr


def test_new_report_endpoints_empty_for_unknown_company(ocr):
    body = ocr.get("/ocr/company/NOBODY/verify-extraction", headers=KEY).json()
    assert body["total_pages"] == 0 and body["pages"] == []
    assert ocr.get("/ocr/company/NOBODY/extracted-cells", headers=KEY).json() == {}


@pytest.mark.parametrize("path", ["verify-extraction", "extracted-cells"])
def test_new_report_endpoints_validate_fiscal_year(ocr, path):
    r = ocr.get(f"/ocr/company/AAA/{path}", headers=KEY, params={"fiscal_year": "../x"})
    assert r.status_code == 422


def test_extracted_cells_and_verification(seeded):
    cells = seeded.get("/ocr/company/SEEDCO/extracted-cells", headers=KEY).json()
    assert set(cells) == {"2024-25_5", "2023-24_7"}
    audit = seeded.get("/ocr/company/SEEDCO/verify-extraction", headers=KEY,
                       params={"fiscal_year": "2024-25"}).json()
    assert audit["total_pages"] == 1 and audit["pages"][0]["expected_cells"] == 2


@pytest.mark.parametrize("md,expected", [
    # Number-like labels (section numbers, years) in the label column are not values.
    ("| Particulars | Note | 2025 | 2024 |\n|---|---|---|---|\n"
     "| **50.1** | **Tax calculation** | | |\n| 2024 | | 10 | 20 |\n| Total | 5 | 30 | 40 |", 4),
    ("| Note | Particulars | 2025 |\n|---|---|---|\n| 12.1 | 3.5 | 7 |", 1),
    ("| Description | 2025 |\n|---|---|\n| 2.3 | 9 |", 1),
    # No label or note column: a numeric first column still counts.
    ("| 2025 | 2024 |\n|---|---|\n| 10 | 20 |", 2),
])
def test_label_column_not_counted_as_value(md, expected):
    from app.ocr.services.extraction_verifier import analyze_markdown_tables
    assert analyze_markdown_tables(md)["total_value_cells"] == expected


def test_results_include_restatement_dots(seeded):
    body = full_results(seeded, "SEEDCO")
    assert body["has_raw_csv"] and "raw_csv_text" not in body
    raw_row = next(r for r in body["raw_rows"] if r["field"].lower() == "total assets")
    assert raw_row["restatement_dots"] == {}

    # Cell details endpoint still provides the restatement dots
    row = merge_and_find(seeded, "SEEDCO", "Total assets")
    cd = seeded.get("/ocr/company/SEEDCO/cell-details", headers=KEY,
                    params={"series_id": row["series_id"], "year": 2024}).json()
    assert [e["type"] for e in cd["flags"]] == ["RESTATED_LOW"]


def test_download_raw_built_from_metrics(seeded):
    r = seeded.get("/ocr/company/SEEDCO/download/raw", headers=KEY)
    assert r.status_code == 200 and "Total assets" in r.text


def test_cell_details_new_schema(seeded):
    row = merge_and_find(seeded, "SEEDCO", "Total assets")
    r = seeded.get("/ocr/company/SEEDCO/cell-details", headers=KEY, params={
        "series_id": row["series_id"], "year": 2024})
    assert r.status_code == 200
    body = r.json()
    assert len(body["cell_records"]) == 2
    assert [e["type"] for e in body["flags"]] == ["RESTATED_LOW"]
    assert [r["restatement_type"] for r in body["cell_records"]] == [None, "RESTATED_LOW"] and "dot_color" not in body["cell_records"][0]
    assert "timeline_summary" not in body and "restatement_dots" not in body
    assert {rec["scope"] for rec in body["cell_records"]} == {"CONSOLIDATED"}


def test_cell_details_needs_a_known_series(seeded):
    assert seeded.get("/ocr/company/SEEDCO/cell-details", headers=KEY, params={"year": 2024}).status_code == 422
    assert seeded.get("/ocr/company/SEEDCO/cell-details", headers=KEY,
                      params={"series_id": 999999, "year": 2024}).status_code == 404


# ---------------------------------------------------------------- pipelines
def test_ocr_pipeline_without_document_or_image_is_404(ocr):
    r = ocr.post("/ocr/pipeline/ocr", headers=KEY, json={
        "instrument_code": "NOPDF", "fiscal_year": "2024-25", "page_numbers": [2, 1]})
    assert r.status_code == 404
    assert "not registered" in r.json()["detail"]


def _seed_page_images(code, fy, pages):
    from app.ocr import database
    out = ocr_config.PAGES_DIR / code / fy
    out.mkdir(parents=True, exist_ok=True)
    for num in pages:
        img = out / f"page_{num}.png"
        img.write_bytes(b"\x89PNG\r\n\x1a\n")
        database.upsert_page(None, code, fy, num, str(img), is_selected=True)


def test_ocr_pipeline_dedupes_pages_and_reports_status(ocr, monkeypatch):
    from app.ocr.services import ocr_service

    def failing_ocr(*_a, **_k):
        raise RuntimeError("no gemini in tests")

    monkeypatch.setattr(ocr_service, "run_gemini_ocr_on_image", failing_ocr)
    _seed_page_images("IMGCO", "2024-25", [1, 2, 3])
    r = ocr.post("/ocr/pipeline/ocr", headers=KEY, json={
        "instrument_code": "IMGCO", "fiscal_year": "2024-25", "page_numbers": [2, 1, 2, 3]})
    body = r.json()
    assert r.status_code == 200
    assert [p["page_number"] for p in body["pages"]] == [2, 1, 3]
    assert all(p["extraction_status"] == "PENDING" for p in body["pages"])
    assert "see server logs" in body["pages"][0]["ocr_markdown"]


def test_extract_pipeline_verifies_and_retries(ocr, monkeypatch):
    from app.ocr.services import extraction_service

    _seed_page("RETRYCO", "2024-25", 3, _MD, [])
    calls = []

    def fake_extract(md, company_name, security_code, model_name, corrective_hint=None, **_kw):
        calls.append(corrective_hint)
        items = [_item("Total assets", [(2025, 1000), (2024, 900)])] if corrective_hint else []
        return {"items": items, "tokens": 10, "duration_sec": 0.1,
                "prompt_tokens": 6, "candidates_tokens": 4}

    monkeypatch.setattr(extraction_service, "extract_financial_data_from_markdown", fake_extract)
    body = ocr.post("/ocr/pipeline/extract", headers=KEY, json={
        "instrument_code": "RETRYCO", "fiscal_year": "2024-25", "page_numbers": [3]}).json()

    assert calls[0] is None and calls[1].startswith("CRITICAL")
    (res,) = body["verification_results"]
    assert res["is_success"] and res["verification_status"] == "VERIFIED"
    assert res["expected_cells"] == res["extracted_cells"] == 2
    assert res["tokens"] == 20 and res["prompt_tokens"] == 12
    page = next(p for p in ocr.get("/ocr/company/RETRYCO/year/2024-25/pages", headers=KEY).json()["pages"]
                if p["page"] == 3)
    assert page["verification_status"] == "VERIFIED" and page["completeness_pct"] == 100.0


def test_parallel_page_renders_of_one_pdf_do_not_fail(tmp_path, monkeypatch):
    import concurrent.futures
    import pypdfium2 as pdfium
    from app.ocr.services import ocr_service
    pdf = pdfium.PdfDocument.new()
    for _ in range(8):
        pdf.new_page(200, 200)
    path = tmp_path / "doc.pdf"
    pdf.save(str(path))
    pdf.close()
    monkeypatch.setattr(ocr_service, "PAGES_DIR", tmp_path / "pages")
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        out = list(ex.map(lambda n: ocr_service.render_single_pdf_page(str(path), "PARCO", "2024-25", n), range(1, 9)))
    assert [o["page_number"] for o in out] == list(range(1, 9))
