rpx_benchmark.reports

  1"""Result serialisation: JSON + markdown summary for benchmark runs."""
  2
  3from __future__ import annotations
  4
  5import json
  6from dataclasses import asdict, is_dataclass
  7from pathlib import Path
  8from typing import Any, Dict
  9
 10from .deployment import DeploymentReadinessReport
 11from .evaluators import BenchmarkResult
 12
 13
 14def _to_jsonable(obj: Any) -> Any:
 15    if is_dataclass(obj):
 16        return _to_jsonable(asdict(obj))
 17    if isinstance(obj, dict):
 18        return {k: _to_jsonable(v) for k, v in obj.items()}
 19    if isinstance(obj, (list, tuple)):
 20        return [_to_jsonable(v) for v in obj]
 21    if hasattr(obj, "value"):  # Enum
 22        return obj.value
 23    return obj
 24
 25
 26def write_json(
 27    path: str | Path,
 28    *,
 29    task: str,
 30    model_name: str,
 31    split: str,
 32    repo_id: str,
 33    result: BenchmarkResult,
 34    dr_report: DeploymentReadinessReport | None = None,
 35    extra: Dict[str, Any] | None = None,
 36) -> Path:
 37    """Serialise a benchmark result + deployment report to JSON.
 38
 39    Parameters
 40    ----------
 41    path : str or Path
 42        Output file path. Parent directories are created if missing.
 43    task : str
 44        Task name string (e.g. ``"monocular_depth"``).
 45    model_name : str
 46        Display name of the model under test.
 47    split : str
 48        ESD difficulty split (``"easy"``, ``"medium"``, ``"hard"``).
 49    repo_id : str
 50        HuggingFace dataset repo id the samples came from.
 51    result : BenchmarkResult
 52        Per-sample + aggregated metric container returned by
 53        :class:`~rpx_benchmark.runner.BenchmarkRunner`.
 54    dr_report : DeploymentReadinessReport, optional
 55        Weighted Phase Score, STR, TS, efficiency metadata. Omitted
 56        when ``None``.
 57    extra : dict, optional
 58        Arbitrary free-form extra fields to embed under the ``extra``
 59        key of the payload. Useful for run-specific metadata a caller
 60        wants preserved alongside the standard keys.
 61
 62    Returns
 63    -------
 64    Path
 65        The resolved output path, for chaining.
 66
 67    Notes
 68    -----
 69    Dataclasses are converted via :func:`dataclasses.asdict`, enums
 70    are lowered to their ``.value``, and unknown objects pass through
 71    unchanged. The output is pretty-printed with indent=2 for
 72    diff-friendliness.
 73    """
 74    path = Path(path)
 75    path.parent.mkdir(parents=True, exist_ok=True)
 76    payload: Dict[str, Any] = {
 77        "task": task,
 78        "model": model_name,
 79        "split": split,
 80        "repo_id": repo_id,
 81        "num_samples": result.num_samples,
 82        "aggregated": result.aggregated,
 83        "per_sample": result.per_sample,
 84    }
 85    if dr_report is not None:
 86        # Split the report into the three top-level RPX axes (no
 87        # composite score; see benchmark/README.md for the policy).
 88        dr_json = _to_jsonable(dr_report)
 89        payload["robustness"] = {
 90            "weighted_phase_score": dr_json.get("weighted_phase_score"),
 91            "temporal_stability":   dr_json.get("temporal_stability"),
 92            "state_transition":     dr_json.get("state_transition"),
 93            "geometric_coherence":  dr_json.get("geometric_coherence"),
 94        }
 95        payload["compute_cost"] = {
 96            "params_m":              dr_json.get("params_m"),
 97            "flops_g":               dr_json.get("flops_g"),
 98            "macs_g":                dr_json.get("macs_g"),
 99            "actmem_gb_fp16":        dr_json.get("actmem_gb_fp16"),
100            "memory_traffic_gb":     dr_json.get("memory_traffic_gb"),
101            "arithmetic_intensity":  dr_json.get("arithmetic_intensity"),
102            "roofline":              dr_json.get("roofline"),
103            "latency_ms_per_sample": dr_json.get("latency_ms_per_sample"),
104            "peak_memory_mb":        dr_json.get("peak_memory_mb"),
105            "system_card":           dr_json.get("system_card"),
106            "operating_point":       dr_json.get("operating_point"),
107        }
108    if extra:
109        payload["extra"] = extra
110    with path.open("w", encoding="utf-8") as f:
111        json.dump(_to_jsonable(payload), f, indent=2)
112    return path
113
114
115def format_markdown_summary(
116    *,
117    task: str,
118    model_name: str,
119    split: str,
120    repo_id: str,
121    result: BenchmarkResult,
122    dr_report: DeploymentReadinessReport | None = None,
123) -> str:
124    """Render a benchmark result as a human-readable markdown report.
125
126    Parameters
127    ----------
128    task : str
129    model_name : str
130    split : str
131    repo_id : str
132    result : BenchmarkResult
133    dr_report : DeploymentReadinessReport, optional
134        When provided, the output includes the Weighted Phase Score
135        table, State-Transition Robustness deltas, Temporal Stability
136        score, and an Efficiency table (params, FLOPs, latency).
137
138    Returns
139    -------
140    str
141        A Markdown-formatted report. Matches the terminal UI tables
142        the CLI prints so on-disk reports and terminal output stay in
143        sync.
144
145    Examples
146    --------
147    >>> from rpx_benchmark.reports import format_markdown_summary  # doctest: +SKIP
148    >>> md = format_markdown_summary(                              # doctest: +SKIP
149    ...     task="monocular_depth", model_name="depth_pro",
150    ...     split="hard", repo_id="IRVLUTD/RPX",
151    ...     result=result, dr_report=report,
152    ... )
153    """
154    lines = [
155        f"# RPX benchmark — {task}",
156        "",
157        f"- **Model:** `{model_name}`",
158        f"- **Split:** `{split}`",
159        f"- **Repo:** `{repo_id}`",
160        f"- **Samples:** {result.num_samples}",
161        "",
162        "## Aggregated metrics",
163        "",
164        "| metric | value |",
165        "|---|---|",
166    ]
167    for k, v in result.aggregated.items():
168        lines.append(f"| {k} | {v:.4f} |")
169
170    if dr_report is not None and dr_report.weighted_phase_score is not None:
171        wps = dr_report.weighted_phase_score
172        lines += [
173            "",
174            "## Weighted Phase Score",
175            "",
176            "| phase | score |",
177            "|---|---|",
178            f"| clutter | {wps.s_clutter:.4f} |",
179            f"| interaction | {wps.s_interaction:.4f} |",
180            f"| clean | {wps.s_clean:.4f} |",
181            f"| **overall** | **{wps.s_overall:.4f}** |",
182            f"| Δ interaction (S_I − S_C) | {wps.delta_int:+.4f} |",
183            f"| Δ recovery    (S_L − S_I) | {wps.delta_rec:+.4f} |",
184        ]
185        if dr_report.state_transition is not None:
186            st = dr_report.state_transition
187            lines += [
188                "",
189                f"- **STR C→I (interaction drop):** {st.str_c_to_i:+.4f}",
190                f"- **STR I→L (recovery):**         {st.str_i_to_l:+.4f}",
191            ]
192        if dr_report.temporal_stability is not None:
193            lines += [
194                f"- **Temporal stability (TS):** {dr_report.temporal_stability.ts_score:.4f}",
195            ]
196
197    if dr_report is not None:
198        eff_rows = []
199        if dr_report.params_m is not None:
200            eff_rows.append(("params (M)", f"{dr_report.params_m:.2f}"))
201        if dr_report.flops_g is not None:
202            eff_rows.append(("FLOPs (G)", f"{dr_report.flops_g:.2f}"))
203        if dr_report.latency_ms_per_sample is not None:
204            eff_rows.append(("latency (ms/sample)", f"{dr_report.latency_ms_per_sample:.1f}"))
205        if dr_report.peak_memory_mb is not None:
206            eff_rows.append(("peak memory (MB)", f"{dr_report.peak_memory_mb:.1f}"))
207        if eff_rows:
208            lines += ["", "## Efficiency", "", "| metric | value |", "|---|---|"]
209            for k, v in eff_rows:
210                lines.append(f"| {k} | {v} |")
211
212    return "\n".join(lines) + "\n"