rpx_benchmark.metrics.specs

  1"""Per-metric specifications used by JEDI and Φ.
  2
  3A :class:`MetricSpec` carries the *direction* (higher-is-better vs lower-is-
  4better), the per-metric *bounds* ``(best, worst)``, and whether those bounds
  5are theoretical (e.g., AP ∈ [0,1]) or empirically frozen across a fixed model
  6zoo. Specs are registered alongside the metric calculators that produce the
  7numeric values; downstream code (JEDI in :mod:`rpx_benchmark.jedi`, Φ in
  8:mod:`rpx_benchmark.phi`) consults the spec registry rather than hard-coding
  9direction or bounds.
 10
 11The design follows paper §3.3: direction-aware bounds avoid the "negate
 12lower-is-better metrics" foot-gun, and the frozen-observed bounds policy
 13makes JEDI reproducible across re-evaluations.
 14
 15Usage
 16-----
 17
 18Calculators expose their specs via a ``specs()`` classmethod; the metric
 19runtime imports the module, the ``@register_metric`` decorator runs, and
 20specs are auto-registered. Direct lookup::
 21
 22    from rpx_benchmark.metrics import get_spec
 23    spec = get_spec("absrel")
 24    assert spec.direction == "lower"
 25"""
 26
 27from __future__ import annotations
 28
 29from dataclasses import dataclass
 30from typing import Dict, Iterable, Literal
 31
 32from ..exceptions import MetricError
 33from ..logging_utils import get_logger
 34
 35log = get_logger(__name__)
 36
 37
 38Direction = Literal["higher", "lower"]
 39
 40
 41@dataclass(frozen=True)
 42class MetricSpec:
 43    """Static metadata about a single scalar metric.
 44
 45    Attributes
 46    ----------
 47    name : str
 48        Key under which the metric is emitted by the calculator
 49        (e.g. ``"absrel"``, ``"ap50"``). Must match the dict key that
 50        :meth:`MetricCalculator.compute` returns.
 51    direction : {"higher", "lower"}
 52        Whether higher raw values indicate better performance.
 53    best : float
 54        The optimal end of the metric scale. For higher-is-better
 55        metrics this is the maximum; for lower-is-better, the minimum.
 56    worst : float
 57        The non-optimal end. For higher-is-better, the minimum; for
 58        lower-is-better, the maximum.
 59    theoretical : bool
 60        True iff (best, worst) are derived from the metric's
 61        mathematical definition (e.g. AP ∈ [0, 1]). False if they are
 62        empirically frozen over a model zoo. The distinction is
 63        documented per paper §3.3 — theoretical bounds are stable
 64        forever; empirical bounds are part of a versioned release.
 65    description : str
 66        One-line human-readable summary; printed in the JEDI/Φ output
 67        cards for debugging.
 68    """
 69
 70    name: str
 71    direction: Direction
 72    best: float
 73    worst: float
 74    theoretical: bool
 75    description: str = ""
 76
 77    def __post_init__(self) -> None:
 78        if self.direction not in ("higher", "lower"):
 79            raise MetricError(
 80                f"MetricSpec direction must be 'higher' or 'lower', got {self.direction!r}",
 81                hint="See rpx_benchmark.metrics.specs.Direction.",
 82            )
 83        if self.direction == "higher" and self.best <= self.worst:
 84            raise MetricError(
 85                f"MetricSpec({self.name!r}): direction=higher requires best > worst, "
 86                f"got best={self.best}, worst={self.worst}",
 87            )
 88        if self.direction == "lower" and self.best >= self.worst:
 89            raise MetricError(
 90                f"MetricSpec({self.name!r}): direction=lower requires best < worst, "
 91                f"got best={self.best}, worst={self.worst}",
 92            )
 93
 94
 95# --------------------------------------------------------------------------- #
 96# Registry
 97# --------------------------------------------------------------------------- #
 98
 99_SPECS: Dict[str, MetricSpec] = {}
100
101
102def register_spec(spec: MetricSpec) -> MetricSpec:
103    """Register a :class:`MetricSpec` under its ``name``.
104
105    Idempotent: re-registering the same name with an *identical* spec is a
106    silent no-op; re-registering with a *different* spec raises
107    :class:`MetricError` (to catch accidental overwrites during module
108    reloads).
109    """
110    existing = _SPECS.get(spec.name)
111    if existing is not None and existing != spec:
112        raise MetricError(
113            f"MetricSpec for {spec.name!r} already registered with different values; "
114            f"existing={existing}, new={spec}",
115            hint="Either match the existing spec or unregister it first.",
116        )
117    _SPECS[spec.name] = spec
118    return spec
119
120
121def register_specs(specs: Iterable[MetricSpec]) -> None:
122    """Convenience wrapper to register a batch."""
123    for s in specs:
124        register_spec(s)
125
126
127def get_spec(name: str) -> MetricSpec:
128    """Look up the spec for a metric ``name``; raise if unknown."""
129    try:
130        return _SPECS[name]
131    except KeyError as e:
132        raise MetricError(
133            f"No MetricSpec registered for {name!r}",
134            hint=(
135                "Import the metric module that owns this name "
136                "(e.g. `import rpx_benchmark.metrics.depth`) so its specs "
137                "auto-register, or call register_spec(...) explicitly."
138            ),
139        ) from e
140
141
142def has_spec(name: str) -> bool:
143    """True iff a spec is registered under ``name``."""
144    return name in _SPECS
145
146
147def registered_specs() -> Dict[str, MetricSpec]:
148    """Snapshot of the current spec registry (defensive copy)."""
149    return dict(_SPECS)
150
151
152def clear_registry() -> None:
153    """Remove every registered spec. Primarily for tests."""
154    _SPECS.clear()
155
156
157# --------------------------------------------------------------------------- #
158# Built-in specs for every shipped task
159# --------------------------------------------------------------------------- #
160#
161# Bounds policy (paper §3.3):
162#   - theoretical=True  → derived from the metric's mathematical definition
163#                          (e.g. AP_50 ∈ [0, 1]).
164#   - theoretical=False → frozen observed bounds across the shipped model
165#                          zoo. The numbers below are the v1 freeze; rerunning
166#                          on a new model set requires an explicit re-release
167#                          (see §3.3 paragraph on out-of-bounds clipping).
168#
169# When a new model produces a score outside (worst, best), the individual
170# desirability d_k clips to [0, 1] and the toolkit flags the cell
171# `out-of-bounds`. Bounds are never silently re-fit.
172
173# Depth (Image Depth frame-level & Video Depth video) — robotics-first set.
174#
175# Organised into three blocks (see experiments/depth_benchmark/METRICS_FINAL.md):
176#   A. Standard accuracy   — citable, general-perception comparability.
177#   B. Robotics deployment — near-range, grasp-tolerance 3D, surface geometry.
178#   C. Temporal (video)    — per-clip stability; lower-is-better except TCC.
179#
180# Range-stratified variants (e.g. "absrel_near"/"_mid"/"_far") are NOT
181# registered individually: the summary layer resolves a suffixed name to its
182# base spec by stripping the trailing _near/_mid/_far token.
183#
184# Intentionally EXCLUDED from the final set (feasibility, see METRICS_FINAL.md):
185#   - relnormal : 1M-Sobol curvature consistency — highest cost-to-signal.
186#   - irmse     : inverse-depth RMSE — low added info over AbsRel/SILog on D435.
187# delta2/delta3 stay registered (the calculator still emits them) but are
188# reported as secondary; delta1 is the primary threshold metric.
189#
190# Bounds tagged "provisional v0" are placeholders to be FROZEN after the v1
191# model-zoo run (cell_log → frozen empirical bounds). Theoretical bounds
192# (fractions ∈ [0,1]) are stable forever.
193DEPTH_SPECS = (
194    # A. Standard accuracy — metric mode; relative models evaluated post-alignment
195    MetricSpec("absrel",    "lower",  best=0.0,  worst=0.5,  theoretical=False,
196               description="Absolute relative depth error |d̂−d|/d"),
197    MetricSpec("rmse",      "lower",  best=0.0,  worst=2.0,  theoretical=False,
198               description="Root-mean-square depth error (metres)"),
199    MetricSpec("rmselog",   "lower",  best=0.0,  worst=0.5,  theoretical=False,
200               description="RMSE in log-depth space (provisional v0)"),
201    MetricSpec("silog",     "lower",  best=0.0,  worst=25.0, theoretical=False,
202               description="Scale-invariant log error Var(log d̂−log d) (provisional v0)"),
203    MetricSpec("delta1",    "higher", best=1.0,  worst=0.0,  theoretical=True,
204               description="Fraction of pixels with max(d̂/d, d/d̂) < 1.25 (primary)"),
205    MetricSpec("delta2",    "higher", best=1.0,  worst=0.0,  theoretical=True,
206               description="Threshold accuracy < 1.25² (secondary)"),
207    MetricSpec("delta3",    "higher", best=1.0,  worst=0.0,  theoretical=True,
208               description="Threshold accuracy < 1.25³ (secondary)"),
209    # B. Robotics deployment — near-range, grasp-tolerance 3D, surface geometry
210    MetricSpec("chamfer_l1",    "lower",  best=0.0,  worst=0.5,  theoretical=False,
211               description="Camera-frame Chamfer-L1 of back-projected cloud, metres — pose-free (provisional v0)"),
212    MetricSpec("fscore_5cm",    "higher", best=1.0,  worst=0.0,  theoretical=True,
213               description="Grasp-reachability F-score @ 5cm, camera-frame. Also report @1cm/@2cm."),
214    MetricSpec("normal_mae",    "lower",  best=0.0,  worst=45.0, theoretical=False,
215               description="Mean angular error of depth-derived normals, degrees — grasp orientation (provisional v0)"),
216    MetricSpec("normal_acc1125","higher", best=1.0,  worst=0.0,  theoretical=True,
217               description="Fraction of surface normals within 11.25° of GT"),
218    MetricSpec("boundary_f1",   "higher", best=1.0,  worst=0.0,  theoretical=True,
219               description="Scale-invariant occluding-contour F1 from depth ratios — relative-only on holey GT"),
220    # C. Temporal (Video Depth only; per phase clip)
221    MetricSpec("opw",  "lower",  best=0.0, worst=0.1, theoretical=False,
222               description="Optical-flow warping consistency, RAFT flow. Not in RPX K-vector (external flow model dependency; RGB-D-only policy). Diagnostic only (provisional v0)"),
223    MetricSpec("tae",  "lower",  best=0.0, worst=0.1, theoretical=False,
224               description="Temporal alignment error via SE(3) reprojection. Not in RPX K-vector (needs T265 poses; RGB-D-only policy). Diagnostic only (provisional v0)"),
225    MetricSpec("tgm",  "lower",  best=0.0, worst=0.1, theoretical=False,
226               description="Temporal gradient matching vs GT depth (provisional v0)"),
227    MetricSpec("tcc",  "higher", best=1.0, worst=0.0, theoretical=True,
228               description="Temporal consistency (SSIM on depth-change maps)"),
229    MetricSpec("tgse", "lower",  best=0.0, worst=0.1, theoretical=False,
230               description="Temporal gradient squared error (L2, sign-preserving; VDPP, arXiv:2604.06665)"),
231    MetricSpec("tmc",  "higher", best=1.0, worst=0.0, theoretical=True,
232               description="Temporal motion consistency (SSIM on depth-flow maps; Zhang ICCV 2019)"),
233    # C.1 Range-stratified temporal (per depth bin)
234    MetricSpec("tae_near", "lower", best=0.0, worst=0.1, theoretical=False,
235               description="TAE at near range (0.3-1.0m, tabletop manipulation)"),
236    MetricSpec("tae_mid",  "lower", best=0.0, worst=0.1, theoretical=False,
237               description="TAE at mid range (1.0-2.5m, arm's reach)"),
238    MetricSpec("tae_far",  "lower", best=0.0, worst=0.1, theoretical=False,
239               description="TAE at far range (2.5-5.0m, room-scale)"),
240    # C.2 Range-stratified spatial accuracy (per depth bin, clip-averaged)
241    MetricSpec("absrel_near", "lower",  best=0.0, worst=1.0, theoretical=False,
242               description="AbsRel at near range (0.3-1.0m)"),
243    MetricSpec("absrel_mid",  "lower",  best=0.0, worst=1.0, theoretical=False,
244               description="AbsRel at mid range (1.0-2.5m)"),
245    MetricSpec("absrel_far",  "lower",  best=0.0, worst=1.0, theoretical=False,
246               description="AbsRel at far range (2.5-5.0m)"),
247    MetricSpec("rmse_near", "lower",  best=0.0, worst=1.0, theoretical=False,
248               description="RMSE at near range (0.3-1.0m)"),
249    MetricSpec("rmse_mid",  "lower",  best=0.0, worst=1.0, theoretical=False,
250               description="RMSE at mid range (1.0-2.5m)"),
251    MetricSpec("rmse_far",  "lower",  best=0.0, worst=1.0, theoretical=False,
252               description="RMSE at far range (2.5-5.0m)"),
253    MetricSpec("delta1_near", "higher", best=1.0, worst=0.0, theoretical=True,
254               description="delta1 at near range (0.3-1.0m)"),
255    MetricSpec("delta1_mid",  "higher", best=1.0, worst=0.0, theoretical=True,
256               description="delta1 at mid range (1.0-2.5m)"),
257    MetricSpec("delta1_far",  "higher", best=1.0, worst=0.0, theoretical=True,
258               description="delta1 at far range (2.5-5.0m)"),
259    # C.3 Range-stratified temporal (per depth bin, pose-free / flow-free)
260    MetricSpec("tgm_near", "lower", best=0.0, worst=0.1, theoretical=False,
261               description="TGM at near range (0.3-1.0m, tabletop manipulation)"),
262    MetricSpec("tgm_mid",  "lower", best=0.0, worst=0.1, theoretical=False,
263               description="TGM at mid range (1.0-2.5m, arm's reach)"),
264    MetricSpec("tgm_far",  "lower", best=0.0, worst=0.1, theoretical=False,
265               description="TGM at far range (2.5-5.0m, room-scale)"),
266    MetricSpec("tgse_near", "lower", best=0.0, worst=0.1, theoretical=False,
267               description="TGSE at near range (0.3-1.0m, tabletop manipulation)"),
268    MetricSpec("tgse_mid",  "lower", best=0.0, worst=0.1, theoretical=False,
269               description="TGSE at mid range (1.0-2.5m, arm's reach)"),
270    MetricSpec("tgse_far",  "lower", best=0.0, worst=0.1, theoretical=False,
271               description="TGSE at far range (2.5-5.0m, room-scale)"),
272)
273
274# Detection / Grounding (D2) — K=4
275DETECTION_SPECS = (
276    MetricSpec("ap50",    "higher", best=1.0, worst=0.0, theoretical=True,
277               description="AP at IoU≥0.50"),
278    MetricSpec("ap75",    "higher", best=1.0, worst=0.0, theoretical=True,
279               description="AP at IoU≥0.75"),
280    MetricSpec("map",     "higher", best=1.0, worst=0.0, theoretical=True,
281               description="mean AP averaged over IoU thresholds"),
282    MetricSpec("ar",      "higher", best=1.0, worst=0.0, theoretical=True,
283               description="Average recall"),
284)
285
286# Tracking (D3) — K=4
287TRACKING_SPECS = (
288    MetricSpec("mota",    "higher", best=1.0, worst=0.0, theoretical=True,
289               description="Multi-Object Tracking Accuracy"),
290    MetricSpec("idf1",    "higher", best=1.0, worst=0.0, theoretical=True,
291               description="ID F1 (identity matching F-score)"),
292    MetricSpec("hota",    "higher", best=1.0, worst=0.0, theoretical=True,
293               description="Higher-Order Tracking Accuracy"),
294    MetricSpec("id_sw",   "lower",  best=0.0, worst=100.0, theoretical=False,
295               description="Identity switches per scene (count)"),
296)
297
298# Scene QA (D4) and Spatial QA (D5) — K=2 each
299QA_SPECS = (
300    MetricSpec("fuzzy_acc",   "higher", best=1.0, worst=0.0, theoretical=True,
301               description="Normalised-similarity fuzzy match accuracy"),
302    MetricSpec("exact_match", "higher", best=1.0, worst=0.0, theoretical=True,
303               description="Strict string-equality accuracy"),
304)
305
306# Relative Camera Pose (D6) — K=4
307POSE_SPECS = (
308    MetricSpec("auc_5deg",   "higher", best=1.0, worst=0.0, theoretical=True,
309               description="AUC of rotation error CDF up to 5°"),
310    MetricSpec("auc_10deg",  "higher", best=1.0, worst=0.0, theoretical=True,
311               description="AUC of rotation error CDF up to 10°"),
312    MetricSpec("auc_20deg",  "higher", best=1.0, worst=0.0, theoretical=True,
313               description="AUC of rotation error CDF up to 20°"),
314    MetricSpec("metric_auc", "higher", best=1.0, worst=0.0, theoretical=True,
315               description="Joint rotation × metric-translation AUC"),
316)
317
318
319
320def _register_builtins() -> None:
321    """Populate the registry with every shipped task's specs at import time."""
322    for batch in (
323        DEPTH_SPECS,
324        DETECTION_SPECS,
325        TRACKING_SPECS,
326        QA_SPECS,
327        POSE_SPECS,
328    ):
329        register_specs(batch)
330
331
332_register_builtins()
333
334
335__all__ = [
336    "Direction",
337    "MetricSpec",
338    "register_spec",
339    "register_specs",
340    "get_spec",
341    "has_spec",
342    "registered_specs",
343    "clear_registry",
344    "DEPTH_SPECS",
345    "DETECTION_SPECS",
346    "TRACKING_SPECS",
347    "QA_SPECS",
348    "POSE_SPECS",
349]