"""Measure real LLM decode/prefill throughput on THIS machine's GPU via ollama.

First-party collector for the K1 dataset (RTX 5080 local-AI throughput matrix,
ops/data-candidates-2026-08-12.md). Design mirrors the price collectors:
mechanical acceptance rules, full raw preservation, a publish gate that stays
closed until enough verified rows exist, and a tracked dated series file.

What it measures, per (model x context) cell, using ollama's own native
counters (no wall-clock guessing):

* decode tok/s  = eval_count / eval_duration
* prefill tok/s = prompt_eval_count / prompt_eval_duration
* load_ms, total_ms, and resident VRAM after generation (nvidia-smi)

Reproducibility block captured with every run: GPU name + driver, ollama
version, model digest + parameter/quant metadata from /api/show, and the
exact generation options (temperature 0, fixed seed, fixed num_predict).

Publish gate: a cell is series-eligible only when it has >= --reps successful
runs; the recorded value is the MEDIAN across reps. publish_ready stays false
at the capture level until the basket coverage floor (--min-cells) is met.
"""

from __future__ import annotations

import argparse
import csv
import json
import statistics
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

OLLAMA = "http://127.0.0.1:11434"
METHOD_VERSION = "llm-throughput-ollama-v1"
_HERE = Path(__file__).resolve()
# In the site repository this file sits at ops/scripts/data/. A standalone copy
# (the one published at /data/rtx-5080-llm-throughput-raw/) writes under the working directory.
REPO = (_HERE.parents[3]
        if len(_HERE.parents) > 3 and (_HERE.parents[3] / "hugo.toml").is_file()
        else Path.cwd())
SERIES_PATH = REPO / "ops" / "data-series" / "llm-throughput-rtx5080.csv"
PUBLISHED_PATH = REPO / "static" / "data" / "rtx-5080-llm-throughput.csv"
SERIES_FIELDS = [
    "capture_date_utc", "method_version", "gpu", "driver",
    "ollama_version", "model", "model_digest12", "parameter_size",
    "quantization", "num_ctx", "prompt_tokens", "reps",
    "median_decode_tok_s", "median_prefill_tok_s", "median_load_ms",
    "vram_resident_mib",
]

# One fixed, boring paragraph repeated to reach the target prompt size keeps
# prefill measurements comparable across models and days.
FILLER = (
    "A home lab is a small collection of computers and network equipment "
    "used to learn, test, and run services. Typical workloads include "
    "virtual machines, containers, storage, backups, and monitoring. "
)


def utc_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def api(path: str, payload: dict[str, Any] | None = None, timeout: int = 600) -> dict[str, Any]:
    req = urllib.request.Request(
        OLLAMA + path,
        data=None if payload is None else json.dumps(payload).encode("utf-8"),
        headers={"Content-Type": "application/json"},
        method="GET" if payload is None else "POST",
    )
    # Ollama's runner can die during a model-swap load ("llama-server process
    # has terminated"), surfacing as a transient 500. Retry twice with a
    # backoff; measurement counters only ever come from a successful response,
    # so retries do not change what a row means (method version unchanged).
    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as response:
                return json.load(response)
        except urllib.error.HTTPError as exc:
            if exc.code >= 500 and attempt < 2:
                time.sleep(15)
                continue
            raise
    raise RuntimeError("unreachable")


def nvidia_query() -> dict[str, Any]:
    try:
        out = subprocess.run(
            ["nvidia-smi", "--query-gpu=name,driver_version,memory.used",
             "--format=csv,noheader,nounits"],
            capture_output=True, text=True, timeout=15, check=True,
        ).stdout.strip().splitlines()[0]
        name, driver, used = [part.strip() for part in out.split(",")]
        return {"gpu": name, "driver": driver, "vram_used_mib": int(float(used))}
    except Exception as exc:  # capture must record, not crash
        return {"gpu": "unavailable", "driver": "unavailable",
                "vram_used_mib": None, "error": str(exc)}


def model_info(model: str) -> dict[str, Any]:
    show = api("/api/show", {"model": model})
    details = show.get("details") or {}
    digest = ""
    for entry in (api("/api/tags").get("models") or []):
        if entry.get("name") == model or entry.get("model") == model:
            digest = str(entry.get("digest") or "")
            break
    return {
        "model": model,
        "digest": digest,
        "digest12": digest[:12],
        "parameter_size": details.get("parameter_size"),
        "quantization": details.get("quantization_level"),
        "family": details.get("family"),
    }


MIN_STABLE_DECODE_TOKENS = 96


def build_prompt(target_tokens: int, nonce: str) -> str:
    # ~4 chars/token heuristic only sizes the filler; actual prompt token
    # counts come back from ollama's prompt_eval_count and are recorded.
    # The per-rep nonce PREFIX forces a fresh prefill every repetition —
    # without it ollama serves warm reps from the prompt cache and the
    # "prefill" rate measures a cache hit (observed: 62k tok/s), not compute.
    words = FILLER * max(1, (target_tokens * 4) // len(FILLER) + 1)
    return (
        f"[capture {nonce}] " + words[: target_tokens * 4]
        + "\nContinue writing this text in the same style, at length. "
        + "Do not summarize, do not conclude, just keep going."
    )


def run_cell(model: str, num_ctx: int, prompt_tokens: int, reps: int,
             num_predict: int) -> dict[str, Any]:
    print(f"cell {model} ctx={num_ctx} ptoks={prompt_tokens} reps={reps}",
          flush=True)
    runs: list[dict[str, Any]] = []
    for rep in range(reps):
        prompt = build_prompt(prompt_tokens, f"{utc_now()}-rep{rep}")
        res = api("/api/generate", {
            "model": model,
            "prompt": prompt,
            "stream": False,
            "options": {
                "num_ctx": num_ctx,
                "temperature": 0,
                "seed": 42,
                "num_predict": num_predict,
            },
        })
        eval_count = res.get("eval_count") or 0
        eval_ns = res.get("eval_duration") or 0
        p_count = res.get("prompt_eval_count") or 0
        p_ns = res.get("prompt_eval_duration") or 0
        # Decode rate is stable once enough tokens were generated; a model may
        # legitimately stop at EOS before the num_predict cap.
        decode_ok = eval_count >= min(num_predict, MIN_STABLE_DECODE_TOKENS) and eval_ns > 0
        # Prefill only counts when the full prompt was actually evaluated
        # (a cache hit reports a tiny prompt_eval_count or absurd rate).
        prefill_ok = p_ns > 0 and p_count >= prompt_tokens * 0.5
        runs.append({
            "rep": rep,
            "decode_ok": decode_ok,
            "prefill_ok": prefill_ok,
            "decode_tok_s": round(eval_count / (eval_ns / 1e9), 2) if eval_ns else None,
            "prefill_tok_s": round(p_count / (p_ns / 1e9), 2) if p_ns else None,
            "prompt_tokens_actual": p_count,
            "output_tokens": eval_count,
            "load_ms": round((res.get("load_duration") or 0) / 1e6, 1),
            "total_ms": round((res.get("total_duration") or 0) / 1e6, 1),
        })
    decode_good = [run for run in runs if run["decode_ok"]]
    prefill_good = [run for run in runs if run["prefill_ok"]]
    cell: dict[str, Any] = {
        "num_ctx": num_ctx,
        "prompt_tokens_target": prompt_tokens,
        "reps_requested": reps,
        "reps_decode_ok": len(decode_good),
        "reps_prefill_ok": len(prefill_good),
        "runs": runs,
        "series_eligible": len(decode_good) >= reps and len(prefill_good) >= 1,
    }
    if decode_good:
        cell["median_decode_tok_s"] = round(
            statistics.median(run["decode_tok_s"] for run in decode_good), 2)
        cell["median_load_ms"] = round(
            statistics.median(run["load_ms"] for run in decode_good), 1)
        cell["prompt_tokens_actual"] = decode_good[0]["prompt_tokens_actual"]
    if prefill_good:
        cell["median_prefill_tok_s"] = round(
            statistics.median(run["prefill_tok_s"] for run in prefill_good), 2)
    cell["vram_after"] = nvidia_query()
    return cell


def append_series(capture: dict[str, Any], path: Path = SERIES_PATH) -> int:
    rows: list[dict[str, Any]] = []
    date = capture["captured_at"][:10]
    env = capture["environment"]
    for model_block in capture["models"]:
        info = model_block["model_info"]
        for cell in model_block["cells"]:
            if not cell.get("series_eligible"):
                continue
            rows.append({
                "capture_date_utc": date,
                "method_version": capture["method_version"],
                "gpu": env["gpu"],
                "driver": env["driver"],
                "ollama_version": env["ollama_version"],
                "model": info["model"],
                "model_digest12": info["digest12"],
                "parameter_size": info["parameter_size"],
                "quantization": info["quantization"],
                "num_ctx": cell["num_ctx"],
                "prompt_tokens": cell.get("prompt_tokens_actual"),
                "reps": cell["reps_decode_ok"],
                "median_decode_tok_s": cell["median_decode_tok_s"],
                "median_prefill_tok_s": cell["median_prefill_tok_s"],
                "median_load_ms": cell["median_load_ms"],
                "vram_resident_mib": (cell.get("vram_after") or {}).get("vram_used_mib"),
            })
    if not rows:
        return 0
    path.parent.mkdir(parents=True, exist_ok=True)
    seen: set[tuple[str, str, str]] = set()
    exists = path.exists()
    if exists:
        with path.open("r", encoding="utf-8", newline="") as handle:
            for existing in csv.DictReader(handle):
                seen.add((existing["capture_date_utc"], existing["model"],
                          existing["num_ctx"]))
    written = 0
    with path.open("a", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=SERIES_FIELDS)
        if not exists:
            writer.writeheader()
        for row in rows:
            key = (row["capture_date_utc"], row["model"], str(row["num_ctx"]))
            if key in seen:
                continue
            writer.writerow(row)
            written += 1
    if written:
        PUBLISHED_PATH.parent.mkdir(parents=True, exist_ok=True)
        PUBLISHED_PATH.write_bytes(path.read_bytes())
    return written


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--models", nargs="+", required=True)
    parser.add_argument("--contexts", nargs="+", type=int, default=[4096])
    parser.add_argument("--prompt-tokens", nargs="+", type=int, default=[512])
    parser.add_argument("--reps", type=int, default=3)
    parser.add_argument("--num-predict", type=int, default=256)
    parser.add_argument("--min-cells", type=int, default=9,
                        help="capture-level publish floor")
    parser.add_argument("--out-dir", type=Path, default=(
        REPO / "ops" / "audit" / "evidence"
        / datetime.now(timezone.utc).strftime("%Y-%m-%d")
        / "llm-throughput-rtx5080"
    ))
    args = parser.parse_args(argv)

    version = api("/api/version").get("version")
    env = {**nvidia_query(), "ollama_version": version}
    capture: dict[str, Any] = {
        "schema": "techfuelhq:llm-throughput-capture:1",
        "method_version": METHOD_VERSION,
        "captured_at": utc_now(),
        "environment": env,
        "options": {"temperature": 0, "seed": 42,
                    "num_predict": args.num_predict},
        "models": [],
    }
    for model in args.models:
        block = {"model_info": model_info(model), "cells": []}
        for ctx in args.contexts:
            for ptoks in args.prompt_tokens:
                if ptoks + args.num_predict > ctx:
                    continue
                block["cells"].append(
                    run_cell(model, ctx, ptoks, args.reps, args.num_predict))
        capture["models"].append(block)

    eligible = sum(
        1 for blk in capture["models"] for cell in blk["cells"]
        if cell.get("series_eligible"))
    capture["quality_gate"] = {
        "publish_ready": False,
        "minimum_cells": args.min_cells,
        "eligible_cells": eligible,
        "blocking_reasons": [
            "Dataset page publication requires operator review of methodology.",
        ] + ([f"Only {eligible} eligible cells; minimum is {args.min_cells}."]
             if eligible < args.min_cells else []),
    }

    args.out_dir.mkdir(parents=True, exist_ok=True)
    out = args.out_dir / f"capture-{capture['captured_at'][:19].replace(':', '')}.json"
    out.write_text(json.dumps(capture, indent=1), encoding="utf-8")
    written = append_series(capture)
    print(f"cells eligible={eligible}; series rows appended={written}; raw={out}")
    return 0


if __name__ == "__main__":
    sys.exit(main())
