rpx_benchmark.tasks.video_depth

  1"""Video absolute depth (paper task Video Depth).
  2
  3Per-clip metric depth from a full ``(scene, phase)`` RGB sequence. The
  4Video Depth roster (DepthCrafter, ChronoDepth, RollingDepth, MonST3R, VGGT-Ω,
  5DA3, etc.) consumes the whole clip in one inference call and emits a
  6per-frame depth sequence.
  7
  8The end-to-end runner is :func:`run_video_pipeline`
  9(:mod:`rpx_benchmark.tasks._video_pipeline`); this module wraps it
 10with the Video Depth task spec and registers it in the task registry.
 11"""
 12
 13from __future__ import annotations
 14
 15from dataclasses import dataclass
 16
 17from ..api import TaskType
 18from ._pipeline import PipelineResult
 19from ._video_pipeline import VideoTaskRunConfig, run_video_pipeline
 20from .registry import TaskSpec, register_task
 21
 22PRIMARY_METRIC = "absrel"
 23
 24# --------------------------------------------------------------------------- #
 25# MANOVA K-vectors (paper §3.2)
 26# --------------------------------------------------------------------------- #
 27#
 28# The K-vector is the set of metrics standardised across all 3N (scene, phase)
 29# cells before running the repeated-measures MANOVA that yields Φ. K must stay
 30# safely below N (scenes) and metrics inside must not be collinear or the
 31# within-phase SSCP E becomes singular. See :func:`rpx_benchmark.phi.compute_phi_oneway`.
 32#
 33# Locked K-vectors — RGB-D-only constraint (see benchmark/README.md):
 34#
 35# Depth-estimation tasks in RPX use RGB-D only. Even though the dataset
 36# captures T265 poses, fisheye stereo, and other modalities, the depth
 37# tasks intentionally exclude them — both from model input and from
 38# metric inputs. This rules out any metric that needs poses (TAE, ATE)
 39# or external models (OPW/RAFT flow, MFC/Sintel flow).
 40#
 41#   * Image Depth K = 5: (absrel, rmse, delta1, silog, fscore_5cm).
 42#       Grasp F@5cm needs only GT depth + D435 intrinsics, so it stays.
 43#
 44#   * Video Depth K = 6: (absrel, rmse, delta1, silog, tgm, tgse).
 45#       Two temporal metrics — TGM (L1 of |Δd_pred|−|Δd_gt| in static
 46#       regions) and TGSE (L2 signed variant) — both pose-free and
 47#       flow-free, needing only pred + GT depth. fscore_5cm drops from
 48#       the video K-vector into diagnostics because the temporal-slot
 49#       tradeoff prioritises the differentiating axis for video models.
 50#
 51# TAE, OPW, fscore_5cm (for D1-V), TCC, TMC, δ₂/δ₃, and range-stratified
 52# variants remain registered as diagnostics — the calculators still emit
 53# them where possible, they just don't enter Φ.
 54FRAME_DEPTH_MANOVA_METRICS: tuple[str, ...] = (
 55    "absrel", "rmse", "delta1", "silog", "fscore_5cm",
 56)
 57VIDEO_DEPTH_MANOVA_METRICS: tuple[str, ...] = (
 58    "absrel", "rmse", "delta1", "silog", "tgm", "tgse",
 59)
 60
 61# Paper-facing aliases (D1-F / D1-V naming used in tables and figures).
 62D1F_MANOVA_METRICS = FRAME_DEPTH_MANOVA_METRICS
 63D1V_MANOVA_METRICS = VIDEO_DEPTH_MANOVA_METRICS
 64
 65D1F_TASK = "monocular_depth"
 66D1V_TASK = VIDEO_DEPTH_TASK = "video_depth"
 67
 68
 69@dataclass
 70class VideoDepthRunConfig(VideoTaskRunConfig):
 71    """Knobs for a Video Depth run.
 72
 73    Inherits `rpx_benchmark.tasks._video_pipeline.VideoTaskRunConfig.frame_budget`
 74    and `rpx_benchmark.tasks._video_pipeline.VideoTaskRunConfig.sampling` from
 75    :class:`~rpx_benchmark.tasks._video_pipeline.VideoTaskRunConfig` for
 76    the temporal-resolution ablation (paper §5.2); leave the defaults
 77    (``frame_budget=None, sampling="all"``) for the headline run.
 78    """
 79
 80
 81def run_video_depth(cfg: VideoDepthRunConfig) -> PipelineResult:
 82    """End-to-end Video Depth pipeline.
 83
 84    Delegates to :func:`run_video_pipeline`, which handles download →
 85    :class:`~rpx_benchmark.video_loader.VideoDepthDataset` build → per-clip
 86    predict → per-clip ``(s, t)`` alignment (for relative-depth
 87    models) →
 88    :class:`~rpx_benchmark.metrics.video_depth.VideoDepthErrorMetrics`
 89    + :class:`~rpx_benchmark.metrics.video_depth.VideoDepthTemporalMetrics`
 90    → cell-log row → JSON / markdown summary.
 91    """
 92    return run_video_pipeline(
 93        task=TaskType.VIDEO_DEPTH,
 94        primary_metric=PRIMARY_METRIC,
 95        cfg=cfg,
 96    )
 97
 98
 99def evaluate_video_depth_clip(pred_seq, clip, *, flow_fn=None):
100    """Score metric-depth predictions for a PhaseClip without downloading data.
101
102    Pose/flow metrics are diagnostics when those inputs are supplied; the
103    headline RGB-D-only K-vector remains unchanged. Non-release image sizes
104    yield NaN for the fixed-intrinsics paper F-score.
105    """
106    import numpy as np
107
108    from ..exceptions import MetricError
109    from ..metrics.depth_temporal import compute_temporal_depth_metrics
110    from ..metrics.video_depth import _per_frame_error_metrics
111
112    pred = np.asarray(pred_seq, dtype=np.float32)
113    gt = np.asarray(clip.depth_gt_seq, dtype=np.float32)
114    if pred.shape != gt.shape or pred.ndim != 3:
115        raise MetricError(f"Expected depth clip shape {gt.shape}, got {pred.shape}")
116    if np.asarray(clip.valid_mask_seq).shape != gt.shape:
117        raise MetricError("Clip validity mask must match ground-truth depth")
118    metrics = _per_frame_error_metrics(pred, gt, np.asarray(clip.valid_mask_seq, dtype=bool))
119    k = np.asarray(clip.intrinsics)
120    metrics.update(compute_temporal_depth_metrics(
121        pred, gt_seq=gt, poses=clip.poses, rgb_seq=clip.rgb_seq, flow_fn=flow_fn,
122        fx=float(k[0, 0]), fy=float(k[1, 1]), cx=float(k[0, 2]), cy=float(k[1, 2]),
123    ))
124    return metrics
125
126
127def video_depth_cell_row(pred_seq, clip, *, model_name, flow_fn=None):
128    """Evaluate a PhaseClip and return the common per-scene/phase cell schema."""
129    from ..cell_log import cell_from_metrics
130
131    return cell_from_metrics(
132        evaluate_video_depth_clip(pred_seq, clip, flow_fn=flow_fn),
133        model_name=model_name, task=VIDEO_DEPTH_TASK, scene_id=clip.scene_id,
134        phase=clip.phase, n_samples=clip.num_frames, difficulty=clip.difficulty,
135    )
136
137
138evaluate_d1v_clip = evaluate_video_depth_clip
139d1v_cell_row = video_depth_cell_row
140
141
142TASK_SPEC = TaskSpec(
143    task=TaskType.VIDEO_DEPTH,
144    display_name="Video Absolute Depth",
145    description=(
146        "Metric depth (metres) from the full ~250-frame phase RGB clip. "
147        "Differs from MONOCULAR_DEPTH in iteration unit (per-clip) and "
148        "metric set (adds temporal consistency metrics on top of Image Depth's "
149        "per-frame metrics)."
150    ),
151    primary_metric=PRIMARY_METRIC,
152    required_modalities=["rgb", "depth"],
153    higher_is_better=False,
154    run=run_video_depth,
155)
156
157register_task(TASK_SPEC)