rpx_benchmark.metrics.depth_temporal

  1"""Clip-level temporal depth metrics for Video Depth.
  2
  3The per-frame depth calculators in :mod:`rpx_benchmark.metrics.depth`
  4operate on a single ``(prediction, ground_truth)`` pair, which is the
  5right contract for Image Depth (frame-level depth).  Video Depth additionally scores a
  6model on how *temporally consistent* its depth sequence is.  Those
  7metrics need a whole clip (``T`` frames) plus, for some of them, the
  8T265 camera poses — they do not fit the per-sample
  9:class:`~rpx_benchmark.metrics.registry.MetricCalculator` interface, so
 10they live here as standalone sequence functions.
 11
 12Locked MANOVA temporal tuple for Video Depth (``benchmark/README.md``):
 13
 14* ``tae`` — Temporal Alignment Error (pose-based geometric consistency).
 15  In the K = 7 MANOVA vector.
 16* ``opw`` — Optical-Flow Warping error (appearance-based consistency).
 17  In the K = 7 MANOVA vector.  Needs an optical-flow backend (RAFT);
 18  the math here is backend-agnostic, the backend is injected.
 19
 20Emitted for diagnostics but **excluded** from the MANOVA K:
 21
 22* ``tgm`` — Temporal Gradient Matching vs GT depth.
 23* ``tcc`` — Temporal Consistency (SSIM on depth-change maps).
 24
 25Directionality (see :mod:`rpx_benchmark.metrics.specs`): ``tae``, ``opw``,
 26``tgm`` are lower-is-better; ``tcc`` is higher-is-better.
 27
 28All metrics are computed only over valid pixels.  For the GT-referenced
 29metrics (``tgm``, ``tcc``) validity is the standard D435 gate
 30(:func:`~rpx_benchmark.metrics.depth_alignment.default_valid_mask`).
 31For the GT-free metrics (``tae``, ``opw``) validity is "the prediction
 32is positive and finite, and the warp/reprojection landed on a valid
 33pixel" — these measure the prediction's *self*-consistency under known
 34camera motion (``tae``) or scene appearance flow (``opw``).
 35"""
 36
 37from __future__ import annotations
 38
 39from typing import Callable, Dict, Optional
 40
 41import numpy as np
 42
 43from ..deployment import _D435_CX, _D435_CY, _D435_FX, _D435_FY, se3_reproject_depth
 44from ..exceptions import MetricError
 45from ..logging_utils import get_logger
 46from .depth_alignment import default_valid_mask
 47
 48log = get_logger(__name__)
 49
 50__all__ = [
 51    "FlowFn",
 52    "temporal_alignment_error",
 53    "optical_flow_warping_error",
 54    "temporal_gradient_matching",
 55    "temporal_gradient_squared_error",
 56    "temporal_motion_consistency",
 57    "temporal_consistency_coefficient",
 58    "compute_temporal_depth_metrics",
 59    "range_stratified_tae",
 60    "range_stratified_tgm_tgse",
 61    "range_stratified_per_frame_metrics",
 62    "TEMPORAL_GRADIENT_STATIC_THRESH_M",
 63    "DEPTH_RANGE_BINS",
 64]
 65
 66#: A flow backend maps two RGB frames (H, W, 3) to a dense flow field
 67#: (H, W, 2) in pixels, expressed as the *backward* flow from the second
 68#: frame to the first: a pixel ``(y, x)`` in ``rgb_to`` corresponds to
 69#: ``(y + flow[y, x, 1], x + flow[y, x, 0])`` in ``rgb_from``.  This is
 70#: the convention :func:`optical_flow_warping_error` warps with.  RAFT's
 71#: forward pass produces exactly this when called as ``raft(rgb_to,
 72#: rgb_from)``.
 73FlowFn = Callable[[np.ndarray, np.ndarray], np.ndarray]
 74
 75#: TGM only compares depth-change in *static* regions (|ΔGT| below this),
 76#: per the appendix definition — dynamic pixels carry real scene motion,
 77#: not model jitter.  0.05 m matches Video Depth Anything (Chen et al. 2025).
 78TEMPORAL_GRADIENT_STATIC_THRESH_M: float = 0.05
 79
 80#: A clip shorter than this is not scored — temporal metrics are not
 81#: meaningful on a handful of frames (decisions doc §3.8).
 82MIN_CLIP_FRAMES: int = 30
 83
 84
 85# --------------------------------------------------------------------------- #
 86# Small numeric helpers
 87# --------------------------------------------------------------------------- #
 88
 89
 90def _absrel(pred: np.ndarray, ref: np.ndarray, valid: np.ndarray) -> Optional[float]:
 91    """Mean ``|pred − ref| / ref`` over ``valid`` pixels, or ``None`` if empty."""
 92    if not valid.any():
 93        return None
 94    p = pred[valid].astype(np.float64)
 95    r = ref[valid].astype(np.float64)
 96    return float(np.mean(np.abs(p - r) / r))
 97
 98
 99def _bilinear_sample(img: np.ndarray, ys: np.ndarray, xs: np.ndarray) -> np.ndarray:
100    """Bilinearly sample ``img`` (H, W) at float coords ``ys``/``xs``.
101
102    Coordinates outside the image are clamped to the border.  Returns an
103    array shaped like ``ys``.
104    """
105    h, w = img.shape
106    x0 = np.floor(xs).astype(np.int64)
107    y0 = np.floor(ys).astype(np.int64)
108    x1 = x0 + 1
109    y1 = y0 + 1
110
111    wx = xs - x0
112    wy = ys - y0
113
114    x0c = np.clip(x0, 0, w - 1)
115    x1c = np.clip(x1, 0, w - 1)
116    y0c = np.clip(y0, 0, h - 1)
117    y1c = np.clip(y1, 0, h - 1)
118
119    img64 = img.astype(np.float64)
120    va = img64[y0c, x0c]
121    vb = img64[y0c, x1c]
122    vc = img64[y1c, x0c]
123    vd = img64[y1c, x1c]
124
125    top = va * (1 - wx) + vb * wx
126    bot = vc * (1 - wx) + vd * wx
127    return top * (1 - wy) + bot * wy
128
129
130def _ssim(a: np.ndarray, b: np.ndarray) -> float:
131    """SSIM between two equal-shaped arrays.
132
133    Uses scikit-image when available (the community-standard impl) and
134    falls back to a global single-window SSIM otherwise, so the metric is
135    always computable without a hard skimage dependency.
136    """
137    try:
138        from skimage.metrics import structural_similarity  # type: ignore
139
140        rng = float(max(a.max() - a.min(), b.max() - b.min(), 1e-6))
141        return float(structural_similarity(a, b, data_range=rng))
142    except Exception:  # pragma: no cover - exercised only without skimage
143        a64 = a.astype(np.float64).ravel()
144        b64 = b.astype(np.float64).ravel()
145        mu_a, mu_b = a64.mean(), b64.mean()
146        va, vb = a64.var(), b64.var()
147        cov = float(np.mean((a64 - mu_a) * (b64 - mu_b)))
148        c1 = (0.01 * 7.0) ** 2
149        c2 = (0.03 * 7.0) ** 2
150        num = (2 * mu_a * mu_b + c1) * (2 * cov + c2)
151        den = (mu_a**2 + mu_b**2 + c1) * (va + vb + c2)
152        return float(num / den) if den else 0.0
153
154
155def _as_seq(name: str, arr: np.ndarray, ndim: int) -> np.ndarray:
156    a = np.asarray(arr)
157    if a.ndim != ndim:
158        raise MetricError(
159            f"{name} must be a {ndim}-D sequence, got shape {a.shape}.",
160            hint="Expected (T, H, W) for depth/mask sequences, (T, H, W, 3) for RGB.",
161        )
162    if a.shape[0] < 2:
163        raise MetricError(f"{name} needs at least 2 frames, got T={a.shape[0]}.")
164    return a
165
166
167# --------------------------------------------------------------------------- #
168# TAE — Temporal Alignment Error (pose-based, GT-free).  In MANOVA K.
169# --------------------------------------------------------------------------- #
170
171
172def temporal_alignment_error(
173    pred_seq: np.ndarray,
174    poses: np.ndarray,
175    *,
176    fx: float = _D435_FX,
177    fy: float = _D435_FY,
178    cx: float = _D435_CX,
179    cy: float = _D435_CY,
180) -> float:
181    """Bidirectional SE(3)-reprojection AbsRel of a predicted depth clip.
182
183    For every consecutive frame pair ``(t, t+1)`` the prediction at one
184    frame is back-projected to 3D, transported to the other camera via
185    the known world-from-camera poses, and re-projected.  The depth it
186    *predicts there* is compared (AbsRel) against the model's *own*
187    prediction at that frame.  A geometrically consistent model has
188    ``tae → 0``; a model whose per-frame depth disagrees in 3D under
189    known camera motion scores high.  No ground truth is used (this is a
190    self-consistency measure), matching ChronoDepth / Video Depth
191    Anything.
192
193    Parameters
194    ----------
195    pred_seq : np.ndarray
196        ``(T, H, W)`` predicted depth in metres.
197    poses : np.ndarray
198        ``(T, 4, 4)`` world-from-camera SE(3) per frame.
199    fx, fy, cx, cy : float
200        Pinhole intrinsics (default D435).
201
202    Returns
203    -------
204    float
205        Mean bidirectional AbsRel over all valid consecutive pairs.
206        ``nan`` if no pair yields a single valid reprojected pixel.
207    """
208    pred_seq = _as_seq("pred_seq", pred_seq, 3)
209    poses = np.asarray(poses, dtype=np.float64)
210    if poses.shape[0] != pred_seq.shape[0] or poses.shape[1:] != (4, 4):
211        raise MetricError(
212            f"poses must be (T, 4, 4) with T={pred_seq.shape[0]}, got {poses.shape}.",
213        )
214
215    errors = []
216    for t in range(pred_seq.shape[0] - 1):
217        d0, d1 = pred_seq[t], pred_seq[t + 1]
218        p0, p1 = poses[t], poses[t + 1]
219        if not (np.all(np.isfinite(p0)) and np.all(np.isfinite(p1))):
220            continue
221
222        # Forward: frame t -> frame t+1.
223        reproj01, ok01 = se3_reproject_depth(d0, p0, p1, fx, fy, cx, cy)
224        v01 = ok01 & np.isfinite(reproj01) & (reproj01 > 0) & (d1 > 0) & np.isfinite(d1)
225        e01 = _absrel(reproj01, d1, v01)
226        if e01 is not None:
227            errors.append(e01)
228
229        # Backward: frame t+1 -> frame t.
230        reproj10, ok10 = se3_reproject_depth(d1, p1, p0, fx, fy, cx, cy)
231        v10 = ok10 & np.isfinite(reproj10) & (reproj10 > 0) & (d0 > 0) & np.isfinite(d0)
232        e10 = _absrel(reproj10, d0, v10)
233        if e10 is not None:
234            errors.append(e10)
235
236    return float(np.mean(errors)) if errors else float("nan")
237
238
239# --------------------------------------------------------------------------- #
240# OPW — Optical-Flow Warping error (appearance-based, GT-free).  In MANOVA K.
241# --------------------------------------------------------------------------- #
242
243
244def optical_flow_warping_error(
245    pred_seq: np.ndarray,
246    rgb_seq: np.ndarray,
247    flow_fn: Optional[FlowFn],
248) -> float:
249    """AbsRel between flow-warped consecutive predicted depth maps.
250
251    For each pair ``(t, t+1)`` the backward flow ``rgb[t+1] → rgb[t]`` is
252    used to sample the depth prediction at ``t`` into the pixel grid of
253    ``t+1``; the warped depth is compared (AbsRel) against the prediction
254    at ``t+1``.  This measures appearance-level temporal smoothness and
255    needs no GT or pose — only an optical-flow backend.
256
257    Parameters
258    ----------
259    pred_seq : np.ndarray
260        ``(T, H, W)`` predicted depth in metres.
261    rgb_seq : np.ndarray
262        ``(T, H, W, 3)`` RGB uint8 frames (the flow source).
263    flow_fn : FlowFn | None
264        Backend that returns the **backward** flow (see :data:`FlowFn`).
265        ``None`` raises :class:`MetricError` — OPW cannot be computed
266        without a flow model.  RPX pins RAFT (``raft-sintel.pth``); wire
267        it as ``flow_fn=raft_backward_flow``.
268
269    Returns
270    -------
271    float
272        Mean AbsRel over all valid pairs; ``nan`` if none are valid.
273    """
274    pred_seq = _as_seq("pred_seq", pred_seq, 3)
275    rgb_seq = _as_seq("rgb_seq", rgb_seq, 4)
276    if rgb_seq.shape[0] != pred_seq.shape[0]:
277        raise MetricError(
278            f"rgb_seq T={rgb_seq.shape[0]} must match pred_seq T={pred_seq.shape[0]}.",
279        )
280    if flow_fn is None:
281        raise MetricError(
282            "optical_flow_warping_error requires a flow backend; flow_fn is None.",
283            hint=(
284                "OPW needs dense optical flow. RPX pins RAFT (raft-sintel.pth, "
285                "Teed & Deng 2020). Pass flow_fn=<raft backward-flow callable>. "
286                "See benchmark/README.md §2 (optical-flow "
287                "recommendation)."
288            ),
289        )
290
291    h, w = pred_seq.shape[1:]
292    grid_y, grid_x = np.mgrid[0:h, 0:w].astype(np.float64)
293
294    errors = []
295    for t in range(pred_seq.shape[0] - 1):
296        flow = np.asarray(flow_fn(rgb_seq[t + 1], rgb_seq[t]), dtype=np.float64)
297        if flow.shape != (h, w, 2):
298            raise MetricError(
299                f"flow_fn returned shape {flow.shape}, expected ({h}, {w}, 2).",
300            )
301        src_x = grid_x + flow[..., 0]
302        src_y = grid_y + flow[..., 1]
303        warped = _bilinear_sample(pred_seq[t], src_y, src_x)
304
305        d1 = pred_seq[t + 1]
306        inside = (src_x >= 0) & (src_x <= w - 1) & (src_y >= 0) & (src_y <= h - 1)
307        valid = inside & (warped > 0) & np.isfinite(warped) & (d1 > 0) & np.isfinite(d1)
308        e = _absrel(warped, d1, valid)
309        if e is not None:
310            errors.append(e)
311
312    return float(np.mean(errors)) if errors else float("nan")
313
314
315# --------------------------------------------------------------------------- #
316# TGM — Temporal Gradient Matching (GT-referenced).  Diagnostic, not in K.
317# --------------------------------------------------------------------------- #
318
319
320def temporal_gradient_matching(
321    pred_seq: np.ndarray,
322    gt_seq: np.ndarray,
323    *,
324    static_thresh_m: float = TEMPORAL_GRADIENT_STATIC_THRESH_M,
325) -> float:
326    """Mean L1 between predicted and GT per-pixel depth-change in static regions.
327
328    .. math::
329
330        \\text{TGM} = \\frac{1}{T-1}\\sum_t
331            \\big\\| \\,|\\hat d_{t+1}-\\hat d_t| - |d_{t+1}-d_t|\\, \\big\\|_1
332
333    restricted to pixels where ``|d_{t+1} − d_t| < static_thresh_m`` (the
334    scene is locally static, so any predicted change is model jitter).
335    Flow-free, GT-referenced.  Lower is better.
336
337    Returns ``nan`` if no static valid pixels exist across the clip.
338    """
339    pred_seq = _as_seq("pred_seq", pred_seq, 3)
340    gt_seq = _as_seq("gt_seq", gt_seq, 3)
341    if pred_seq.shape != gt_seq.shape:
342        raise MetricError(
343            f"pred_seq {pred_seq.shape} and gt_seq {gt_seq.shape} must match.",
344        )
345
346    errors = []
347    for t in range(pred_seq.shape[0] - 1):
348        v0 = default_valid_mask(pred_seq[t], gt_seq[t])
349        v1 = default_valid_mask(pred_seq[t + 1], gt_seq[t + 1])
350        valid = v0 & v1
351        if not valid.any():
352            continue
353        d_gt = np.abs(gt_seq[t + 1] - gt_seq[t])
354        d_pred = np.abs(pred_seq[t + 1] - pred_seq[t])
355        static = valid & (d_gt < static_thresh_m)
356        if not static.any():
357            continue
358        errors.append(float(np.mean(np.abs(d_pred[static] - d_gt[static]))))
359
360    return float(np.mean(errors)) if errors else float("nan")
361
362
363# --------------------------------------------------------------------------- #
364# TCC — Temporal Consistency Coefficient (GT-referenced).  Diagnostic, not in K.
365# --------------------------------------------------------------------------- #
366
367
368def temporal_consistency_coefficient(
369    pred_seq: np.ndarray,
370    gt_seq: np.ndarray,
371) -> float:
372    """Mean SSIM between predicted and GT temporal depth-change maps.
373
374    Captures the *structural pattern* of depth change over time rather
375    than its per-pixel magnitude (which TGM measures).  Higher is better,
376    in ``[-1, 1]``.  Returns ``nan`` if no pair has valid pixels.
377    """
378    pred_seq = _as_seq("pred_seq", pred_seq, 3)
379    gt_seq = _as_seq("gt_seq", gt_seq, 3)
380    if pred_seq.shape != gt_seq.shape:
381        raise MetricError(
382            f"pred_seq {pred_seq.shape} and gt_seq {gt_seq.shape} must match.",
383        )
384
385    scores = []
386    for t in range(pred_seq.shape[0] - 1):
387        v0 = default_valid_mask(pred_seq[t], gt_seq[t])
388        v1 = default_valid_mask(pred_seq[t + 1], gt_seq[t + 1])
389        valid = v0 & v1
390        if not valid.any():
391            continue
392        diff_pred = (pred_seq[t + 1] - pred_seq[t]) * valid
393        diff_gt = (gt_seq[t + 1] - gt_seq[t]) * valid
394        scores.append(_ssim(diff_pred.astype(np.float64), diff_gt.astype(np.float64)))
395
396    return float(np.mean(scores)) if scores else float("nan")
397
398
399# --------------------------------------------------------------------------- #
400# TGSE — Temporal Gradient Squared Error (GT-referenced).  Diagnostic.
401# Origin: VDPP (Yoon et al., arXiv 2026, arXiv:2604.06665).
402# --------------------------------------------------------------------------- #
403
404
405def temporal_gradient_squared_error(
406    pred_seq: np.ndarray,
407    gt_seq: np.ndarray,
408) -> float:
409    """Mean squared error of temporal depth gradients (pred vs GT).
410
411    .. math::
412
413        \\text{TGSE} = \\frac{1}{T-1}\\sum_t \\frac{1}{|V_t|}
414            \\sum_{(i,j) \\in V_t}
415            \\left( \\hat d^{(t+1)}_{i,j} - \\hat d^{(t)}_{i,j}
416                   - (d^{(t+1)}_{i,j} - d^{(t)}_{i,j}) \\right)^2
417
418    Unlike TGM: (a) preserves sign of depth change (no inner absolute
419    value), (b) uses L2 so severe flicker dominates the average.
420    Lower is better.  Returns ``nan`` if no pair has valid pixels.
421
422    Reference: Yoon et al. "VDPP: Video Depth Post-Processing for Speed
423    and Scalability" (arXiv 2026, 2604.06665).
424    """
425    pred_seq = _as_seq("pred_seq", pred_seq, 3)
426    gt_seq = _as_seq("gt_seq", gt_seq, 3)
427    if pred_seq.shape != gt_seq.shape:
428        raise MetricError(
429            f"pred_seq {pred_seq.shape} and gt_seq {gt_seq.shape} must match.",
430        )
431
432    errors = []
433    for t in range(pred_seq.shape[0] - 1):
434        v0 = default_valid_mask(pred_seq[t], gt_seq[t])
435        v1 = default_valid_mask(pred_seq[t + 1], gt_seq[t + 1])
436        valid = v0 & v1
437        if not valid.any():
438            continue
439        delta_pred = pred_seq[t + 1][valid] - pred_seq[t][valid]
440        delta_gt = gt_seq[t + 1][valid] - gt_seq[t][valid]
441        errors.append(float(np.mean((delta_pred.astype(np.float64)
442                                     - delta_gt.astype(np.float64)) ** 2)))
443
444    return float(np.mean(errors)) if errors else float("nan")
445
446
447# --------------------------------------------------------------------------- #
448# TMC — Temporal Motion Consistency (GT-referenced).  Diagnostic.
449# Origin: Zhang et al. ICCV 2019 (arXiv:1908.03706).
450# --------------------------------------------------------------------------- #
451
452
453def _dense_flow_on_depth(d0: np.ndarray, d1: np.ndarray) -> np.ndarray:
454    """Compute dense flow between two depth maps using Farneback.
455
456    Returns (H, W, 2) flow field.  Falls back to zero-flow if OpenCV is
457    not available.
458    """
459    try:
460        import cv2  # type: ignore
461    except ImportError:  # pragma: no cover
462        return np.zeros((*d0.shape, 2), dtype=np.float32)
463    # Normalize depth to [0, 255] uint8 for Farneback.
464    def _to_u8(d: np.ndarray) -> np.ndarray:
465        d64 = d.astype(np.float64)
466        lo, hi = np.nanmin(d64), np.nanmax(d64)
467        if hi - lo < 1e-8:
468            return np.zeros_like(d64, dtype=np.uint8)
469        return ((d64 - lo) / (hi - lo) * 255).astype(np.uint8)
470
471    return cv2.calcOpticalFlowFarneback(
472        _to_u8(d0), _to_u8(d1),
473        None,  # type: ignore[arg-type]
474        pyr_scale=0.5, levels=3, winsize=15,
475        iterations=3, poly_n=5, poly_sigma=1.2, flags=0,
476    )
477
478
479def temporal_motion_consistency(
480    pred_seq: np.ndarray,
481    gt_seq: np.ndarray,
482) -> float:
483    """Mean SSIM between optical flows computed on depth maps (pred vs GT).
484
485    Computes dense optical flow (Farneback) on consecutive depth frames
486    for both prediction and GT, then takes the SSIM between the
487    resulting flow fields.  Captures whether the apparent motion
488    *pattern* in depth space matches GT.
489
490    Higher is better, in ``[-1, 1]``.  Returns ``nan`` if no pair is
491    computable or if OpenCV is unavailable.
492
493    Reference: Zhang et al. "Exploiting temporal consistency for
494    real-time video depth estimation" (ICCV 2019, arXiv:1908.03706).
495    """
496    pred_seq = _as_seq("pred_seq", pred_seq, 3)
497    gt_seq = _as_seq("gt_seq", gt_seq, 3)
498    if pred_seq.shape != gt_seq.shape:
499        raise MetricError(
500            f"pred_seq {pred_seq.shape} and gt_seq {gt_seq.shape} must match.",
501        )
502
503    scores = []
504    for t in range(pred_seq.shape[0] - 1):
505        flow_pred = _dense_flow_on_depth(pred_seq[t], pred_seq[t + 1])
506        flow_gt = _dense_flow_on_depth(gt_seq[t], gt_seq[t + 1])
507        # SSIM on flow magnitude
508        mag_pred = np.sqrt(flow_pred[..., 0] ** 2 + flow_pred[..., 1] ** 2).astype(np.float64)
509        mag_gt = np.sqrt(flow_gt[..., 0] ** 2 + flow_gt[..., 1] ** 2).astype(np.float64)
510        scores.append(_ssim(mag_pred, mag_gt))
511
512    return float(np.mean(scores)) if scores else float("nan")
513
514
515# --------------------------------------------------------------------------- #
516# Range-stratified TAE and per-frame metrics
517# --------------------------------------------------------------------------- #
518
519#: Depth range bins for robotics-aware stratification (metres).
520#: Near = tabletop manipulation, Mid = arm's reach, Far = room-scale.
521DEPTH_RANGE_BINS = {
522    "near": (0.3, 1.0),
523    "mid": (1.0, 2.5),
524    "far": (2.5, 5.0),
525}
526
527#: Minimum valid-pixel fraction per frame for a bin to be scored.
528_MIN_BIN_PIXELS = 100
529
530
531def range_stratified_tae(
532    pred_seq: np.ndarray,
533    gt_seq: np.ndarray,
534    poses: np.ndarray,
535    *,
536    fx: float = _D435_FX,
537    fy: float = _D435_FY,
538    cx: float = _D435_CX,
539    cy: float = _D435_CY,
540) -> Dict[str, float]:
541    """TAE stratified by GT-depth range bins.
542
543    For each consecutive frame pair and each range bin, computes the
544    TAE only over pixels whose GT depth at the source frame falls in
545    that bin.  Returns ``{tae_near, tae_mid, tae_far}``; bins with
546    insufficient valid pixels are ``nan``.
547    """
548    pred_seq = _as_seq("pred_seq", pred_seq, 3)
549    gt_seq = _as_seq("gt_seq", gt_seq, 3)
550    poses = np.asarray(poses, dtype=np.float64)
551    if poses.shape[0] != pred_seq.shape[0] or poses.shape[1:] != (4, 4):
552        raise MetricError(
553            f"poses must be (T, 4, 4) with T={pred_seq.shape[0]}, got {poses.shape}.",
554        )
555
556    bin_errors: Dict[str, list] = {f"tae_{k}": [] for k in DEPTH_RANGE_BINS}
557
558    for t in range(pred_seq.shape[0] - 1):
559        d0, d1 = pred_seq[t], pred_seq[t + 1]
560        g0 = gt_seq[t]
561        p0, p1 = poses[t], poses[t + 1]
562        if not (np.all(np.isfinite(p0)) and np.all(np.isfinite(p1))):
563            continue
564
565        # Forward reproject: frame t -> frame t+1
566        reproj01, ok01 = se3_reproject_depth(d0, p0, p1, fx, fy, cx, cy)
567        base_valid = ok01 & np.isfinite(reproj01) & (reproj01 > 0) & (d1 > 0) & np.isfinite(d1)
568
569        for bin_name, (lo, hi) in DEPTH_RANGE_BINS.items():
570            bin_mask = base_valid & (g0 >= lo) & (g0 < hi)
571            if bin_mask.sum() < _MIN_BIN_PIXELS:
572                continue
573            e = _absrel(reproj01, d1, bin_mask)
574            if e is not None:
575                bin_errors[f"tae_{bin_name}"].append(e)
576
577    return {
578        k: float(np.mean(v)) if v else float("nan")
579        for k, v in bin_errors.items()
580    }
581
582
583def range_stratified_tgm_tgse(
584    pred_seq: np.ndarray,
585    gt_seq: np.ndarray,
586    *,
587    static_thresh_m: float = TEMPORAL_GRADIENT_STATIC_THRESH_M,
588) -> Dict[str, float]:
589    """TGM and TGSE stratified by GT depth range, clip-averaged.
590
591    For each consecutive frame pair and each range bin, computes:
592      * TGM = mean L1(|Δd_pred| − |Δd_gt|) over static-region pixels
593        (|Δd_gt| < ``static_thresh_m``) whose GT depth at frame ``t``
594        falls in the bin;
595      * TGSE = mean squared error of signed depth-change (Δd_pred − Δd_gt)
596        over all valid pixels whose GT depth at frame ``t`` falls in the
597        bin (no static mask; TGSE is defined for all changes, not just
598        static regions).
599
600    Returns ``{tgm_near, tgm_mid, tgm_far, tgse_near, tgse_mid, tgse_far}``.
601    Bins with fewer than ``_MIN_BIN_PIXELS`` in any pair are ``nan``.
602    """
603    pred_seq = _as_seq("pred_seq", pred_seq, 3)
604    gt_seq = _as_seq("gt_seq", gt_seq, 3)
605    if pred_seq.shape != gt_seq.shape:
606        raise MetricError(
607            f"pred_seq {pred_seq.shape} and gt_seq {gt_seq.shape} must match.",
608        )
609
610    tgm_bins: Dict[str, list] = {f"tgm_{b}": [] for b in DEPTH_RANGE_BINS}
611    tgse_bins: Dict[str, list] = {f"tgse_{b}": [] for b in DEPTH_RANGE_BINS}
612
613    for t in range(pred_seq.shape[0] - 1):
614        v0 = default_valid_mask(pred_seq[t], gt_seq[t])
615        v1 = default_valid_mask(pred_seq[t + 1], gt_seq[t + 1])
616        valid = v0 & v1
617        if not valid.any():
618            continue
619
620        g0 = gt_seq[t]
621        d_gt_abs = np.abs(gt_seq[t + 1] - gt_seq[t])
622        d_pred_abs = np.abs(pred_seq[t + 1] - pred_seq[t])
623        delta_gt = gt_seq[t + 1] - gt_seq[t]
624        delta_pred = pred_seq[t + 1] - pred_seq[t]
625        static_mask = valid & (d_gt_abs < static_thresh_m)
626
627        for bin_name, (lo, hi) in DEPTH_RANGE_BINS.items():
628            bin_source = (g0 >= lo) & (g0 < hi)
629
630            tgm_pix = static_mask & bin_source
631            if tgm_pix.sum() >= _MIN_BIN_PIXELS:
632                tgm_bins[f"tgm_{bin_name}"].append(
633                    float(np.mean(np.abs(d_pred_abs[tgm_pix] - d_gt_abs[tgm_pix])))
634                )
635
636            tgse_pix = valid & bin_source
637            if tgse_pix.sum() >= _MIN_BIN_PIXELS:
638                tgse_bins[f"tgse_{bin_name}"].append(
639                    float(np.mean((delta_pred[tgse_pix] - delta_gt[tgse_pix]) ** 2))
640                )
641
642    out: Dict[str, float] = {}
643    for d in (tgm_bins, tgse_bins):
644        for k, v in d.items():
645            out[k] = float(np.mean(v)) if v else float("nan")
646    return out
647
648
649def range_stratified_per_frame_metrics(
650    pred_seq: np.ndarray,
651    gt_seq: np.ndarray,
652) -> Dict[str, float]:
653    """Per-frame AbsRel/RMSE/δ₁ stratified by GT depth range, clip-averaged.
654
655    Returns keys like ``absrel_near``, ``rmse_mid``, ``delta1_far``.
656    Bins with insufficient valid pixels across the clip are ``nan``.
657    """
658    pred_seq = _as_seq("pred_seq", pred_seq, 3)
659    gt_seq = _as_seq("gt_seq", gt_seq, 3)
660    if pred_seq.shape != gt_seq.shape:
661        raise MetricError(
662            f"pred_seq {pred_seq.shape} and gt_seq {gt_seq.shape} must match.",
663        )
664
665    metric_names = ("absrel", "rmse", "delta1")
666    accum: Dict[str, list] = {}
667    for m in metric_names:
668        for b in DEPTH_RANGE_BINS:
669            accum[f"{m}_{b}"] = []
670
671    for t in range(pred_seq.shape[0]):
672        p = pred_seq[t].astype(np.float64)
673        g = gt_seq[t].astype(np.float64)
674        base_valid = default_valid_mask(pred_seq[t], gt_seq[t])
675
676        for bin_name, (lo, hi) in DEPTH_RANGE_BINS.items():
677            v = base_valid & (g >= lo) & (g < hi)
678            if v.sum() < _MIN_BIN_PIXELS:
679                continue
680            pv, gv = p[v], g[v]
681            accum[f"absrel_{bin_name}"].append(float(np.mean(np.abs(pv - gv) / gv)))
682            accum[f"rmse_{bin_name}"].append(float(np.sqrt(np.mean((pv - gv) ** 2))))
683            ratio = np.maximum(pv / gv, gv / pv)
684            accum[f"delta1_{bin_name}"].append(float(np.mean(ratio < 1.25)))
685
686    return {
687        k: float(np.mean(v)) if v else float("nan")
688        for k, v in accum.items()
689    }
690
691
692# --------------------------------------------------------------------------- #
693# Orchestrator
694# --------------------------------------------------------------------------- #
695
696
697def compute_temporal_depth_metrics(
698    pred_seq: np.ndarray,
699    *,
700    gt_seq: Optional[np.ndarray] = None,
701    poses: Optional[np.ndarray] = None,
702    rgb_seq: Optional[np.ndarray] = None,
703    flow_fn: Optional[FlowFn] = None,
704    fx: float = _D435_FX,
705    fy: float = _D435_FY,
706    cx: float = _D435_CX,
707    cy: float = _D435_CY,
708) -> Dict[str, float]:
709    """Compute every available temporal depth metric for one clip.
710
711    Each metric is computed only when its required inputs are present;
712    missing inputs degrade gracefully (the metric is set to ``nan`` and a
713    note is logged) rather than raising — a clip with no T265 poses still
714    yields ``opw``/``tgm``/``tcc``.
715
716    Inputs
717    ------
718    pred_seq : (T, H, W) predicted depth, metres. Required.
719    gt_seq   : (T, H, W) GT depth, metres. Enables ``tgm`` and ``tcc``.
720    poses    : (T, 4, 4) world-from-camera SE(3). Enables ``tae``.
721    rgb_seq  : (T, H, W, 3) RGB. With ``flow_fn`` enables ``opw``.
722    flow_fn  : optical-flow backend (see :data:`FlowFn`).
723
724    Returns
725    -------
726    dict[str, float]
727        Keys ``tae, opw, tgm, tcc`` (``nan`` where not computable). The
728        MANOVA selection (``tae``, ``opw``) happens downstream in
729        ``analyze_experiment.py``; this calculator emits all four.
730    """
731    out: Dict[str, float] = {
732        "tae": float("nan"), "opw": float("nan"),
733        "tgm": float("nan"), "tcc": float("nan"),
734        "tgse": float("nan"), "tmc": float("nan"),
735    }
736    # Range-stratified keys initialized to nan.
737    for b in DEPTH_RANGE_BINS:
738        out[f"tae_{b}"] = float("nan")
739    for m in ("absrel", "rmse", "delta1", "tgm", "tgse"):
740        for b in DEPTH_RANGE_BINS:
741            out[f"{m}_{b}"] = float("nan")
742
743    if poses is not None:
744        out["tae"] = temporal_alignment_error(pred_seq, poses, fx=fx, fy=fy, cx=cx, cy=cy)
745        if gt_seq is not None:
746            strat = range_stratified_tae(pred_seq, gt_seq, poses, fx=fx, fy=fy, cx=cx, cy=cy)
747            out.update(strat)
748    else:
749        log.debug("temporal: poses absent — tae/tae_stratified skipped (nan)")
750
751    if rgb_seq is not None and flow_fn is not None:
752        out["opw"] = optical_flow_warping_error(pred_seq, rgb_seq, flow_fn)
753    else:
754        log.debug("temporal: rgb_seq/flow_fn absent — opw skipped (nan)")
755
756    if gt_seq is not None:
757        out["tgm"] = temporal_gradient_matching(pred_seq, gt_seq)
758        out["tcc"] = temporal_consistency_coefficient(pred_seq, gt_seq)
759        out["tgse"] = temporal_gradient_squared_error(pred_seq, gt_seq)
760        out["tmc"] = temporal_motion_consistency(pred_seq, gt_seq)
761        strat_pf = range_stratified_per_frame_metrics(pred_seq, gt_seq)
762        out.update(strat_pf)
763        strat_tg = range_stratified_tgm_tgse(pred_seq, gt_seq)
764        out.update(strat_tg)
765    else:
766        log.debug("temporal: gt_seq absent — tgm/tcc/tgse/tmc/stratified skipped (nan)")
767
768    return out