rpx_benchmark.adapters.depth_scaffold

  1"""Shared scaffold for depth-model adapters (Image Depth and Video Depth).
  2
  3Every adapter in ``rpx_benchmark/adapters/depth/`` and
  4``rpx_benchmark/adapters/video_depth/`` follows the same shape:
  5
  6* Declare ``task`` (``TaskType.MONOCULAR_DEPTH`` for Image Depth,
  7  ``TaskType.VIDEO_DEPTH`` for Video Depth).
  8* Declare ``depth_output_kind`` (``"metric"`` or ``"relative"``) so
  9  the runner knows whether to apply per-scene-phase scale-and-shift
 10  alignment before metric computation.
 11* Declare ``install_hint`` so a contributor who hits the
 12  ``NotImplementedError`` knows exactly which dependency to install.
 13* Implement ``setup()`` (lazy weight loading) and ``predict()``
 14  (forward call returning the right Prediction dataclass).
 15
 16The scaffold provides:
 17
 18* A clean :class:`DepthAdapterSkeleton` base class with sensible
 19  defaults and a ``predict`` stub that raises a descriptive
 20  ``NotImplementedError`` until a real implementation lands.
 21* A central :data:`DEPTH_MODEL_CARDS` table that documents every
 22  model in the paper rosters (Image Depth + Video Depth) — name, task, output
 23  kind, paper reference, install hint. Tests assert the table covers
 24  every roster entry so we can't accidentally drop a model.
 25* :func:`available_depth_adapters` — programmatic introspection of
 26  which adapters are importable in the current environment (a model
 27  whose package is uninstalled raises at adapter-class definition
 28  time, so the caller can know up-front).
 29
 30The skeleton approach is deliberate. The 19 models in the rosters
 31have wildly different installation requirements (different CUDA
 32versions, gated HF access, custom forks, multi-GB checkpoints). We
 33ship the registration surface so the harness "knows" about every
 34model from day one, and contributors fill in the forward call when
 35they install each model — no PR-per-model needed once the skeleton
 36exists.
 37"""
 38
 39from __future__ import annotations
 40
 41from dataclasses import dataclass
 42from typing import Any, ClassVar, Literal, Sequence
 43
 44from ..api import BenchmarkModel, Sample, TaskType
 45from ..exceptions import ConfigError, ModelError
 46from ..logging_utils import get_logger
 47
 48log = get_logger(__name__)
 49
 50
 51# --------------------------------------------------------------------------- #
 52# Model cards — the canonical roster
 53# --------------------------------------------------------------------------- #
 54
 55
 56@dataclass(frozen=True)
 57class DepthModelCard:
 58    """Static metadata about one depth model in the roster.
 59
 60    Used by tests to verify the adapter package covers every model
 61    the paper claims to evaluate, and by tooling to render
 62    "available models" tables.
 63    """
 64
 65    name: str
 66    """Display name used in result tables — matches the paper's
 67    Table 2 / Table 3 spelling exactly (e.g. ``"DA3 Metric-L"``)."""
 68
 69    task: TaskType
 70    """``MONOCULAR_DEPTH`` (Image Depth) or ``VIDEO_DEPTH`` (Video Depth)."""
 71
 72    depth_output_kind: Literal["metric", "relative"]
 73    """Determines whether the runner applies per-scene-phase scale +
 74    shift alignment before metrics. ``"metric"`` for models that
 75    output meters directly; ``"relative"`` for affine-invariant
 76    models (Lotus-2, FE2E and DA-V2 relative)."""
 77
 78    install_hint: str
 79    """One-line `pip install ...` (or equivalent) the contributor
 80    runs to make ``setup()`` work. Surfaced in the
 81    ``NotImplementedError`` raised by skeleton ``predict()`` so the
 82    next contributor doesn't have to grep."""
 83
 84    paper_ref: str = ""
 85    """Optional citation key — matches an entry in
 86    ``paper-submission/overleaf/root.bib`` so the model card can
 87    cross-link to its primary source."""
 88
 89
 90#: Canonical roster of every depth model the paper benchmarks.
 91#: Tests assert that every entry here has a corresponding adapter
 92#: skeleton importable from the adapters package.
 93#:
 94#: Source: paper-submission/overleaf/text/neurips_v2/04_tasks.tex
 95#: (the Image Depth and Video Depth rows of Table 2).
 96DEPTH_MODEL_CARDS: dict[str, DepthModelCard] = {
 97    # -------------------------------------------------------------- Image Depth
 98    "da3-metric-l": DepthModelCard(
 99        name="DA3 Metric-L",
100        task=TaskType.MONOCULAR_DEPTH,
101        depth_output_kind="metric",
102        install_hint="pip install depth-anything-3 (or HF transformers + the depth-anything-v3 weights)",
103        paper_ref="yang2024depth",
104    ),
105    "da-v2-large": DepthModelCard(
106        name="DA-V2 Large",
107        task=TaskType.MONOCULAR_DEPTH,
108        depth_output_kind="metric",
109        install_hint="pip install depth-anything-v2 transformers; metric variant from HF",
110        paper_ref="yang2024depthanythingv2",
111    ),
112    "depth-pro": DepthModelCard(
113        name="Depth Pro",
114        task=TaskType.MONOCULAR_DEPTH,
115        depth_output_kind="metric",
116        install_hint="pip install depth-pro (Apple); or `pip install git+https://github.com/apple/ml-depth-pro`",
117        paper_ref="bochkovskii2025depthpro",
118    ),
119    "unidepth-v2": DepthModelCard(
120        name="UniDepth V2",
121        task=TaskType.MONOCULAR_DEPTH,
122        depth_output_kind="metric",
123        install_hint="pip install unidepth (or `pip install git+https://github.com/lpiccinelli-eth/UniDepth`)",
124        paper_ref="piccinelli2024unidepth",
125    ),
126    "metric3d-v2": DepthModelCard(
127        name="Metric3D V2",
128        task=TaskType.MONOCULAR_DEPTH,
129        depth_output_kind="metric",
130        install_hint="pip install metric3d (or `pip install git+https://github.com/YvanYin/Metric3D`)",
131        paper_ref="metric3d_v2",
132    ),
133    "moge-2-vit-l": DepthModelCard(
134        name="MoGe-2 ViT-L",
135        task=TaskType.MONOCULAR_DEPTH,
136        depth_output_kind="metric",
137        install_hint="pip install moge (Microsoft); HF: Ruicheng/moge-2-vitl-normal",
138        paper_ref="wang2024moge",
139    ),
140    "hyden": DepthModelCard(
141        name="HyDen",
142        task=TaskType.MONOCULAR_DEPTH,
143        depth_output_kind="metric",
144        install_hint=(
145            "clone facebookresearch/metadepth; "
146            "HF: facebook/hyden-mogev2-metric-point"
147        ),
148        paper_ref="hyden",
149    ),
150    "lotus-2": DepthModelCard(
151        name="Lotus-2",
152        task=TaskType.MONOCULAR_DEPTH,
153        depth_output_kind="relative",
154        install_hint="pip install diffusers transformers; HF: jingheya/lotus-depth-d-v2",
155        paper_ref="he2024lotus",
156    ),
157    "fe2e": DepthModelCard(
158        name="FE2E",
159        task=TaskType.MONOCULAR_DEPTH,
160        depth_output_kind="relative",
161        install_hint=(
162            "official AMAP-ML/FE2E source plus its linked exander/FE2E "
163            "checkpoint; use the pinned docker/depth-fe2e environment"
164        ),
165        paper_ref="fe2e",
166    ),
167    "zipdepth": DepthModelCard(
168        name="ZipDepth",
169        task=TaskType.MONOCULAR_DEPTH,
170        depth_output_kind="relative",
171        install_hint=(
172            "official fabiotosi92/ZipDepth source and zipdepth_base.pth; "
173            "use the pinned docker/depth-zipdepth environment"
174        ),
175        paper_ref="zipdepth",
176    ),
177    # -------------------------------------------------------------- Video Depth
178    "da3-video": DepthModelCard(
179        name="DA3",
180        task=TaskType.VIDEO_DEPTH,
181        depth_output_kind="relative",
182        install_hint=(
183            "pip install depth-anything-3; depth-anything/DA3-LARGE, "
184            "fed per-clip instead of per-frame"
185        ),
186        paper_ref="yang2024depth",
187    ),
188    "depth-crafter": DepthModelCard(
189        name="DepthCrafter",
190        task=TaskType.VIDEO_DEPTH,
191        depth_output_kind="relative",
192        install_hint="pip install depthcrafter (Tencent); HF: tencent/DepthCrafter",
193        paper_ref="hu2024depthcrafter",
194    ),
195    "video-da": DepthModelCard(
196        name="Video DA",
197        task=TaskType.VIDEO_DEPTH,
198        depth_output_kind="relative",
199        install_hint="pip install video-depth-anything (or upstream repo)",
200        paper_ref="video_depth_anything",
201    ),
202    "chrono-depth": DepthModelCard(
203        name="ChronoDepth",
204        task=TaskType.VIDEO_DEPTH,
205        depth_output_kind="relative",
206        install_hint="pip install chronodepth (TBD — clone upstream repo)",
207        paper_ref="shao2024chronodepth",
208    ),
209    "rolling-depth": DepthModelCard(
210        name="RollingDepth",
211        task=TaskType.VIDEO_DEPTH,
212        depth_output_kind="relative",
213        install_hint="pip install rollingdepth (TBD — clone upstream repo)",
214        paper_ref="rollingdepth",
215    ),
216    "dvd": DepthModelCard(
217        name="DVD v1.1",
218        task=TaskType.VIDEO_DEPTH,
219        depth_output_kind="relative",
220        install_hint=(
221            "clone github.com/EnVision-Research/DVD at the pinned revision; "
222            "official v1.1 weights: huggingface.co/FayeHongfeiZhang/DVD"
223        ),
224        paper_ref="dvd",
225    ),
226    "gem-depth": DepthModelCard(
227        name="GemDepth",
228        task=TaskType.VIDEO_DEPTH,
229        depth_output_kind="relative",
230        install_hint=(
231            "clone github.com/Yuecheng919/GemDepth; "
232            "official weights: huggingface.co/YuechengLiu/GemDepth"
233        ),
234        paper_ref="gemdepth",
235    ),
236    "vigeo": DepthModelCard(
237        name="ViGeo",
238        task=TaskType.VIDEO_DEPTH,
239        depth_output_kind="relative",
240        install_hint=(
241            "clone github.com/aigc3d/ViGeo; "
242            "official weights: huggingface.co/pkqbajng/ViGeo"
243        ),
244        paper_ref="vigeo",
245    ),
246    "monst3r": DepthModelCard(
247        name="MonST3R",
248        task=TaskType.VIDEO_DEPTH,
249        depth_output_kind="relative",
250        install_hint="pip install monst3r (or `pip install git+https://github.com/Junyi42/monst3r`)",
251        paper_ref="zhang2024monst3r",
252    ),
253    "vggt-omega": DepthModelCard(
254        name="VGGT-Ω",
255        task=TaskType.VIDEO_DEPTH,
256        depth_output_kind="relative",
257        install_hint="pip install vggt (or upstream Meta repo); Ω = the large variant",
258        paper_ref="vggt_omega",
259    ),
260}
261
262
263# --------------------------------------------------------------------------- #
264# Skeleton base class
265# --------------------------------------------------------------------------- #
266
267
268class DepthAdapterSkeleton(BenchmarkModel):
269    """Base class for every depth-model adapter skeleton.
270
271    Subclasses set the class-level :attr:`MODEL_KEY` to the
272    ``DEPTH_MODEL_CARDS`` entry they wrap; everything else flows from
273    that card automatically. Override ``setup()`` and ``predict()``
274    with real implementations once the model is installed.
275
276    The default ``predict()`` raises a ``NotImplementedError`` whose
277    message includes the install hint, so any contributor who tries
278    to run an unimplemented adapter sees exactly what they need to do.
279    """
280
281    #: Key into :data:`DEPTH_MODEL_CARDS`. Must be set by subclasses.
282    MODEL_KEY: ClassVar[str] = ""
283
284    def __init_subclass__(cls, **kwargs: Any) -> None:
285        super().__init_subclass__(**kwargs)
286        if not cls.MODEL_KEY:
287            # The abstract scaffold itself doesn't have a key; subclasses
288            # without a key are an error.
289            if cls.__name__ != "DepthAdapterSkeleton":
290                raise ConfigError(
291                    f"{cls.__name__}: MODEL_KEY must be set on every DepthAdapterSkeleton subclass",
292                    hint="Set MODEL_KEY to the matching DEPTH_MODEL_CARDS key.",
293                )
294            return
295        if cls.MODEL_KEY not in DEPTH_MODEL_CARDS:
296            raise ConfigError(
297                f"{cls.__name__}: MODEL_KEY {cls.MODEL_KEY!r} is not in DEPTH_MODEL_CARDS",
298                hint=(
299                    f"Either add a card for {cls.MODEL_KEY!r} to "
300                    f"DEPTH_MODEL_CARDS, or fix the typo. Known keys: "
301                    f"{sorted(DEPTH_MODEL_CARDS)}"
302                ),
303            )
304        card = DEPTH_MODEL_CARDS[cls.MODEL_KEY]
305        # Inherit task + output kind from the card so adapter authors
306        # can't accidentally diverge from the canonical roster.
307        cls.task = card.task
308        cls.depth_output_kind = card.depth_output_kind
309        cls.name = card.name
310        cls.install_hint = card.install_hint
311
312    name: ClassVar[str] = ""
313    install_hint: ClassVar[str] = ""
314
315    def setup(self) -> None:
316        """Load weights, warm CUDA, etc. Skeleton: no-op.
317
318        Override in concrete subclasses. The runner calls this once
319        before iteration; the override should be idempotent so
320        repeated calls (e.g. when warming a multi-process pool) are
321        cheap.
322        """
323        return None
324
325    def predict(self, batch: Sequence[Sample]) -> Sequence[Any]:
326        """Skeleton ``predict`` — raises ``NotImplementedError`` with
327        the install hint baked in so a contributor sees exactly which
328        package to install / which forward call to wire up.
329        """
330        raise NotImplementedError(
331            f"Adapter for {self.name!r} ({self.MODEL_KEY}) is a skeleton. "
332            f"Install the model and implement `predict`: {self.install_hint}",
333        )
334
335
336# --------------------------------------------------------------------------- #
337# Discovery helpers
338# --------------------------------------------------------------------------- #
339
340
341def available_depth_adapters() -> dict[str, type[DepthAdapterSkeleton]]:
342    """Return every DepthAdapterSkeleton subclass currently importable.
343
344    Walks the registered subclasses of :class:`DepthAdapterSkeleton`
345    and returns ``{MODEL_KEY: class}``. Useful for CLI tooling and
346    tests that want to enumerate "every model we've at least
347    skeletoned".
348    """
349    out: dict[str, type[DepthAdapterSkeleton]] = {}
350    for cls in DepthAdapterSkeleton.__subclasses__():
351        if cls.MODEL_KEY:
352            out[cls.MODEL_KEY] = cls
353    return out
354
355
356def lazy_import(module_name: str, *, install_hint: str) -> Any:
357    """Best-effort import with a friendly error.
358
359    Skeleton ``setup()`` overrides call this to defer heavy deps
360    until the adapter is actually run; if the dep is missing, the
361    contributor gets the install hint inline.
362    """
363    try:
364        import importlib
365
366        return importlib.import_module(module_name)
367    except ImportError as e:
368        raise ModelError(
369            f"failed to import {module_name!r}",
370            hint=install_hint,
371        ) from e
372
373
374__all__ = [
375    "DepthAdapterSkeleton",
376    "DepthModelCard",
377    "DEPTH_MODEL_CARDS",
378    "available_depth_adapters",
379    "lazy_import",
380]