rpx_benchmark.metrics.video_depth

  1"""Metric calculators bound to ``TaskType.VIDEO_DEPTH`` (paper task Video Depth).
  2
  3Video Depth scores a model on the same per-frame quality as Image Depth **plus** a
  4small set of temporal-consistency metrics. The implementation already
  5exists in :mod:`rpx_benchmark.metrics.depth` (per-frame error metrics)
  6and :mod:`rpx_benchmark.metrics.depth_temporal` (TAE / OPW / TGM / TCC);
  7this module is the thin glue that registers those metrics for the new
  8``VIDEO_DEPTH`` task and applies them sequence-shape-aware.
  9
 10Cell-log granularity
 11--------------------
 12
 13One :class:`~rpx_benchmark.api.VideoDepthPrediction` corresponds to one
 14``(scene, phase)`` cell — the cell-log key is
 15``(model, task, scene, phase)`` (no per-frame key). Per-frame error
 16metrics are therefore **aggregated to a per-clip mean** before being
 17written to the log. Temporal metrics are inherently per-clip.
 18
 19What the final metric tuple is
 20------------------------------
 21
 22The full set this module exposes is intentionally broader than what
 23the paper's headline Φ MANOVA uses. The final K-metric tuple (5? 6?
 248?) for Video Depth Φ is being decided in Feynman's depth-metric study
 25(:file:`benchmark/README.md`); the downstream
 26selection happens in :mod:`rpx_benchmark.analyze_experiment` from the
 27cell-log column subset, not by deleting calculators here.
 28"""
 29
 30from __future__ import annotations
 31
 32from typing import Dict, Optional
 33
 34import numpy as np
 35
 36from ..api import TaskType, VideoDepthGroundTruth, VideoDepthPrediction
 37from ..exceptions import MetricError
 38from ..logging_utils import get_logger
 39from .depth_alignment import DEPTH_MAX_M, DEPTH_MIN_M
 40from .registry import MetricCalculator, register_metric
 41
 42log = get_logger(__name__)
 43
 44
 45# --------------------------------------------------------------------------- #
 46# Per-frame error metrics, aggregated over the clip
 47# --------------------------------------------------------------------------- #
 48
 49
 50# D435 paper-declared intrinsics for the 640×480 release. Used by the
 51# per-frame F-Score@5cm computation. Frames must be at this resolution
 52# for F-Score to be well-defined; a shape mismatch raises.
 53# Kept in sync with the Image Depth pipeline's ``depth_paper.py``
 54# (Naren's D1-F module on branch jishnu/depth_pipeline_check).
 55_D435_FX = _D435_FY = 615.0
 56_D435_CX = 320.0
 57_D435_CY = 240.0
 58_D435_HW = (480, 640)
 59
 60#: Grasp-tolerance F-Score threshold in metres (paper §5.2).
 61_FSCORE_THRESHOLD_M = 0.05
 62
 63
 64def _backproject(depth: np.ndarray, valid: np.ndarray) -> np.ndarray:
 65    """Back-project (H, W) depth to (N, 3) camera-frame points at valid pixels."""
 66    ys, xs = np.nonzero(valid)
 67    z = depth[valid].astype(np.float64, copy=False)
 68    return np.column_stack(
 69        (
 70            (xs.astype(np.float64) - _D435_CX) * z / _D435_FX,
 71            (ys.astype(np.float64) - _D435_CY) * z / _D435_FY,
 72            z,
 73        )
 74    )
 75
 76
 77def _fscore_5cm_per_frame(
 78    pred: np.ndarray,
 79    gt: np.ndarray,
 80    valid: np.ndarray,
 81) -> Optional[float]:
 82    """Bidirectional cKDTree F-Score at 5 cm — one frame.
 83
 84    Returns ``None`` when the frame has no valid pixels or scipy is
 85    unavailable at the frame level; callers should skip ``None`` before
 86    averaging. Matches the definition in
 87    ``rpx_benchmark.metrics.depth_paper.point_cloud_fscore_5cm``.
 88    """
 89    if not valid.any():
 90        return None
 91    try:
 92        from scipy.spatial import cKDTree
 93    except ImportError:  # pragma: no cover — scipy is a standard dep
 94        return None
 95
 96    gt_pts = _backproject(gt, valid)
 97    pred_pts = _backproject(pred, valid)
 98    if len(gt_pts) == 0 or len(pred_pts) == 0:
 99        return None
100
101    def _matched(tree, pts, chunk: int = 65_536) -> float:
102        matched = 0
103        for start in range(0, len(pts), chunk):
104            d, _ = tree.query(pts[start : start + chunk], k=1, workers=1)
105            matched += int(np.count_nonzero(d < _FSCORE_THRESHOLD_M))
106        return matched / len(pts)
107
108    precision = _matched(cKDTree(gt_pts), pred_pts)
109    recall = _matched(cKDTree(pred_pts), gt_pts)
110    denom = precision + recall
111    return 0.0 if denom == 0.0 else float(2.0 * precision * recall / denom)
112
113
114def _per_frame_error_metrics(
115    pred_seq: np.ndarray,
116    gt_seq: np.ndarray,
117    valid_seq: np.ndarray,
118    *,
119    compute_fscore: bool = True,
120) -> Dict[str, float]:
121    """Per-frame paper-spec metrics → arithmetic mean over the clip.
122
123    Emits the full K=5 paper spatial vector plus δ₂/δ₃ diagnostics:
124    ``absrel``, ``rmse``, ``silog``, ``delta1``, ``fscore_5cm``,
125    ``delta2``, ``delta3``.
126
127    SILog follows the KITTI display convention (``100 × sqrt(Var(log
128    error))``) — matches ``depth_paper.silog`` in the Image Depth pipeline,
129    so cells.parquet values are comparable across Image Depth and Video Depth.
130
131    F@5cm requires 640×480 frames (paper-declared D435 intrinsics).
132    Set ``compute_fscore=False`` to defer the expensive cKDTree pass;
133    the caller can back-fill from saved predictions. Frames whose
134    valid mask is fully zero are skipped from every metric's mean
135    rather than counted as zero.
136    """
137    T = pred_seq.shape[0]
138    per_frame: Dict[str, list] = {
139        "absrel": [],
140        "rmse": [],
141        "silog": [],
142        "delta1": [],
143        "delta2": [],
144        "delta3": [],
145    }
146    fscore_vals: list = []
147    fscore_enabled = compute_fscore and pred_seq.shape[1:] == _D435_HW
148
149    for t in range(T):
150        p = pred_seq[t]
151        g = gt_seq[t]
152        v = (
153            valid_seq[t]
154            & np.isfinite(g)
155            & (g > DEPTH_MIN_M)
156            & (g < DEPTH_MAX_M)
157        )
158        if not v.any():
159            continue
160        if not np.isfinite(p).all():
161            raise MetricError(
162                "Video Depth prediction contains non-finite values",
163                hint=(
164                    "Fix the adapter output. Finite raw predictions are preserved "
165                    "in the cache and uniformly clipped to [0.3, 5.0] only in "
166                    "the evaluation copy."
167                ),
168            )
169        p_eval = np.clip(p, DEPTH_MIN_M, DEPTH_MAX_M)
170        p_v = p_eval[v]
171        g_v = g[v]
172        diff = p_v - g_v
173        log_err = np.log(p_v) - np.log(g_v)
174
175        per_frame["absrel"].append(float(np.mean(np.abs(diff) / g_v)))
176        per_frame["rmse"].append(float(np.sqrt(np.mean(diff ** 2))))
177        silog_var = max(
178            float(np.mean(log_err ** 2) - np.mean(log_err) ** 2), 0.0
179        )
180        per_frame["silog"].append(float(100.0 * np.sqrt(silog_var)))
181        ratio = np.maximum(p_v / g_v, g_v / p_v)
182        per_frame["delta1"].append(float(np.mean(ratio < 1.25)))
183        per_frame["delta2"].append(float(np.mean(ratio < 1.25 ** 2)))
184        per_frame["delta3"].append(float(np.mean(ratio < 1.25 ** 3)))
185
186        if fscore_enabled:
187            fs = _fscore_5cm_per_frame(p_eval, g, v)
188            if fs is not None:
189                fscore_vals.append(fs)
190
191    if not per_frame["absrel"]:
192        raise MetricError(
193            "Video Depth clip contains no GT pixels in the paper-valid "
194            "0.3 < depth < 5.0 m interval",
195        )
196    out: Dict[str, float] = {k: float(np.mean(v)) for k, v in per_frame.items()}
197    out["fscore_5cm"] = (
198        float(np.mean(fscore_vals)) if fscore_vals else float("nan")
199    )
200    return out
201
202
203@register_metric(TaskType.VIDEO_DEPTH)
204class VideoDepthErrorMetrics(MetricCalculator):
205    """Per-clip aggregate of the paper-spec per-frame depth metrics.
206
207    Emits the full K=5 spatial vector plus δ₂/δ₃ diagnostics:
208
209    * ``absrel``, ``rmse``, ``silog`` — spatial error family
210    * ``delta1`` — headline threshold accuracy
211    * ``fscore_5cm`` — 3D grasp-tolerance accuracy (M5 in the paper's
212      locked K=5). Requires 640×480 frames per D435's paper-declared
213      intrinsics.
214    * ``delta2``, ``delta3`` — cross-paper compatibility diagnostics
215      (saturated on SOTA models; kept for Eigen-split tables)
216
217    Each metric is computed per frame and arithmetically averaged over
218    the clip's valid frames. Frames with fully-empty valid masks are
219    skipped from the mean rather than counted as zero.
220
221    SILog follows the KITTI display convention (``100 × sqrt(Var(log
222    error))``), matching ``depth_paper.silog`` in the Image Depth pipeline
223    so cells.parquet values are directly comparable across Image Depth (per-frame)
224    and Video Depth (per-clip average of per-frame) runs.
225    """
226
227    name = "video_depth_error_metrics"
228
229    def compute(
230        self,
231        prediction: VideoDepthPrediction,
232        ground_truth: VideoDepthGroundTruth,
233    ) -> Dict[str, float]:
234        _validate_shapes(prediction, ground_truth)
235        return _per_frame_error_metrics(
236            prediction.depth_map_seq.astype(np.float32),
237            ground_truth.depth_map_seq.astype(np.float32),
238            ground_truth.valid_mask_seq.astype(bool),
239            compute_fscore=bool(ground_truth.compute_fscore),
240        )
241
242
243# --------------------------------------------------------------------------- #
244# Temporal metrics — thin adapter over depth_temporal.py
245# --------------------------------------------------------------------------- #
246
247
248@register_metric(TaskType.VIDEO_DEPTH)
249class VideoDepthTemporalMetrics(MetricCalculator):
250    """Temporal consistency metrics for one clip.
251
252    Calls :func:`~rpx_benchmark.metrics.depth_temporal.compute_temporal_depth_metrics`,
253    which itself dispatches to ``tae``, ``opw``, ``tgm``, ``tcc``
254    depending on which inputs are present. Any metric missing an input
255    (e.g. no T265 poses, no flow function configured) returns ``nan``
256    rather than raising — that matches the paper's reporting policy
257    where the MANOVA's K is fixed downstream from whichever metrics are
258    actually populated.
259
260    Optional sample-level context (poses, RGB, flow function) is read
261    from ``ground_truth.frame_indices``'s sibling fields if the loader
262    attached them. The standard path is the loader populating
263    ``ground_truth.poses`` (T, 4, 4) and ``ground_truth.rgb_seq``
264    (T, H, W, 3) at clip-construction time so this calculator can be
265    stateless.
266    """
267
268    name = "video_depth_temporal"
269
270    def compute(
271        self,
272        prediction: VideoDepthPrediction,
273        ground_truth: VideoDepthGroundTruth,
274    ) -> Dict[str, float]:
275        # Lazy import: ``metrics.depth_temporal`` and its dependencies
276        # (``deployment.se3_reproject_depth``, etc.) are not yet on
277        # ``main`` — they live in a sibling work-in-progress branch.
278        # Importing them eagerly at module load time would break CI on
279        # any PR (like this one) that ships ``video_depth.py`` before
280        # ``depth_temporal.py`` lands. The lazy import lets the
281        # calculator register at task-discovery time and only fail
282        # loudly if a caller actually invokes it before the temporal
283        # module is available.
284        try:
285            from .depth_temporal import compute_temporal_depth_metrics
286        except ImportError:
287            log.warning(
288                "video_depth_temporal: depth_temporal module not "
289                "available on this branch — emitting NaN for all "
290                "temporal metrics. Land depth_temporal.py in a follow-up "
291                "PR to populate these.",
292            )
293            return {"tae": float("nan"), "opw": float("nan"),
294                    "tgm": float("nan"), "tgse": float("nan"),
295                    "tcc": float("nan"), "tmc": float("nan")}
296
297        _validate_shapes(prediction, ground_truth)
298        pred_seq = prediction.depth_map_seq.astype(np.float32)
299        gt_seq = ground_truth.depth_map_seq.astype(np.float32)
300        # Optional extras the loader may have stashed on the GT object
301        # (we read them off the dataclass attribute namespace rather
302        # than baking new required fields into VideoDepthGroundTruth).
303        poses = getattr(ground_truth, "poses", None)
304        rgb_seq = getattr(ground_truth, "rgb_seq", None)
305        flow_fn = getattr(ground_truth, "flow_fn", None)
306        return compute_temporal_depth_metrics(
307            pred_seq,
308            gt_seq=gt_seq,
309            poses=poses,
310            rgb_seq=rgb_seq,
311            flow_fn=flow_fn,
312        )
313
314
315# --------------------------------------------------------------------------- #
316# Shared shape validation
317# --------------------------------------------------------------------------- #
318
319
320def _validate_shapes(
321    prediction: VideoDepthPrediction,
322    ground_truth: VideoDepthGroundTruth,
323) -> None:
324    if not isinstance(prediction, VideoDepthPrediction):
325        raise MetricError(
326            f"VideoDepth calculator expected VideoDepthPrediction, got {type(prediction).__name__}",
327        )
328    if not isinstance(ground_truth, VideoDepthGroundTruth):
329        raise MetricError(
330            f"VideoDepth calculator expected VideoDepthGroundTruth, "
331            f"got {type(ground_truth).__name__}",
332        )
333    if prediction.depth_map_seq.shape != ground_truth.depth_map_seq.shape:
334        raise MetricError(
335            f"VideoDepth prediction shape {prediction.depth_map_seq.shape} "
336            f"does not match ground-truth shape {ground_truth.depth_map_seq.shape}",
337            hint=(
338                "If your adapter subsampled the clip via frame_budget, it "
339                "must return predictions only for those frames AND the "
340                "ground-truth must already be sliced to the same indices "
341                "by VideoDepthDataset (this happens automatically when the loader "
342                "honours frame_indices)."
343            ),
344        )
345    if ground_truth.valid_mask_seq.shape != ground_truth.depth_map_seq.shape:
346        raise MetricError(
347            f"VideoDepth valid_mask_seq shape {ground_truth.valid_mask_seq.shape} "
348            f"does not match depth_map_seq shape {ground_truth.depth_map_seq.shape}",
349        )
350
351
352__all__ = ["VideoDepthErrorMetrics", "VideoDepthTemporalMetrics"]