rpx_benchmark.evaluators

  1"""Legacy metric module (compat shim).
  2
  3``MetricSuite`` and ``BenchmarkResult`` have moved to
  4:mod:`rpx_benchmark.metrics`. This file still re-exports them so old
  5imports continue to work, and it still hosts the raw numpy metric
  6*functions* (``depth_metrics``, ``detection_metrics``, etc.) used
  7internally by the calculator classes.
  8
  9New code should import from :mod:`rpx_benchmark.metrics` instead.
 10"""
 11
 12from __future__ import annotations
 13
 14from typing import Dict, Sequence
 15
 16import numpy as np
 17
 18# Re-export the new registry-backed facade so old ``from
 19# rpx_benchmark.evaluators import MetricSuite`` imports keep working.
 20from .metrics.registry import BenchmarkResult, MetricSuite  # noqa: F401
 21
 22# ------------------------------------------------------------------ #
 23# Task-specific metric functions
 24# ------------------------------------------------------------------ #
 25
 26
 27def depth_metrics(pred: np.ndarray, gt: np.ndarray) -> Dict[str, float]:
 28    """AbsRel, RMSE, and threshold accuracy (δ<1.25) for monocular depth."""
 29    pred = np.asarray(pred, dtype=np.float32)
 30    gt = np.asarray(gt, dtype=np.float32)
 31
 32    valid = gt > 0
 33    if valid.sum() == 0:
 34        return {"rmse": 0.0, "absrel": 0.0, "delta1": 1.0, "delta2": 1.0, "delta3": 1.0}
 35
 36    pred_v, gt_v = pred[valid], gt[valid]
 37    rmse = float(np.sqrt(np.mean((pred_v - gt_v) ** 2)))
 38    absrel = float(np.mean(np.abs(pred_v - gt_v) / gt_v))
 39
 40    thresh = np.maximum(pred_v / gt_v, gt_v / pred_v)
 41    delta1 = float(np.mean(thresh < 1.25))
 42    delta2 = float(np.mean(thresh < 1.25**2))
 43    delta3 = float(np.mean(thresh < 1.25**3))
 44
 45    return {"rmse": rmse, "absrel": absrel, "delta1": delta1, "delta2": delta2, "delta3": delta3}
 46
 47
 48def detection_metrics(
 49    pred_boxes: np.ndarray,
 50    pred_scores: Sequence[float],
 51    pred_labels: Sequence[str],
 52    gt_boxes: np.ndarray,
 53    gt_labels: Sequence[str],
 54    iou_threshold: float = 0.5,
 55) -> Dict[str, float]:
 56    """Greedy matching at a single IoU threshold → precision/recall/F1."""
 57    if len(pred_boxes) == 0 and len(gt_boxes) == 0:
 58        return {"precision": 1.0, "recall": 1.0, "f1": 1.0}
 59    if len(pred_boxes) == 0:
 60        return {"precision": 0.0, "recall": 0.0, "f1": 0.0}
 61
 62    pred_indices = np.argsort(-np.asarray(pred_scores))
 63    matched_gt: set[int] = set()
 64    tp = 0
 65    for idx in pred_indices:
 66        box = pred_boxes[idx]
 67        label = pred_labels[idx]
 68        best_iou, best_j = 0.0, -1
 69        for j, (gt_box, gt_label) in enumerate(zip(gt_boxes, gt_labels, strict=False)):
 70            if j in matched_gt or gt_label != label:
 71                continue
 72            iou = bbox_iou(box, gt_box)
 73            if iou > best_iou:
 74                best_iou, best_j = iou, j
 75        if best_iou >= iou_threshold and best_j >= 0:
 76            matched_gt.add(best_j)
 77            tp += 1
 78    fp = len(pred_boxes) - tp
 79    fn = len(gt_boxes) - tp
 80    precision = tp / (tp + fp) if (tp + fp) else 0.0
 81    recall = tp / (tp + fn) if (tp + fn) else 0.0
 82    f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) else 0.0
 83    return {"precision": float(precision), "recall": float(recall), "f1": float(f1)}
 84
 85
 86def segmentation_metrics(pred_mask: np.ndarray, gt_mask: np.ndarray) -> Dict[str, float]:
 87    """Per-class IoU averaged across classes present in GT (mIoU)."""
 88    pred_mask = np.asarray(pred_mask, dtype=np.int32)
 89    gt_mask = np.asarray(gt_mask, dtype=np.int32)
 90    classes = np.unique(gt_mask)
 91    classes = classes[classes >= 0]  # ignore background = -1 if used
 92
 93    if len(classes) == 0:
 94        return {"miou": 1.0}
 95
 96    ious = []
 97    for c in classes:
 98        pred_c = pred_mask == c
 99        gt_c = gt_mask == c
100        intersection = float((pred_c & gt_c).sum())
101        union = float((pred_c | gt_c).sum())
102        ious.append(intersection / union if union > 0 else 1.0)
103
104    return {"miou": float(np.mean(ious))}
105
106
107def tracking_metrics(
108    pred_tracks: Sequence,
109    gt_tracks: Sequence,
110    iou_threshold: float = 0.5,
111) -> Dict[str, float]:
112    """MOTA and IDF1 for multi-object tracking (single-frame / tracklet level).
113
114    This is a simplified tracklet-level computation suitable for per-sample
115    evaluation across a video clip.  A full MOTA/IDF1 computation requires
116    frame-by-frame association across the full sequence and is typically run
117    by the PhaseStratifiedRunner over an entire scene.
118    """
119    # Build frame-level sets from Tracklet objects
120    # Each Tracklet has boxes: T x 4, so T = number of frames it covers.
121    # We evaluate over each frame index present in GT.
122
123    # Determine number of frames
124    max_t = max((t.boxes.shape[0] for t in gt_tracks), default=0)
125    if max_t == 0:
126        return {"mota": 1.0, "idf1": 1.0}
127
128    total_gt = 0
129    total_fp = 0
130    total_fn = 0
131    total_idsw = 0  # ID switches — approximated here
132
133    # id assignment tracking: pred_track_id -> last matched gt_track_id
134    pred_last_match: Dict[str, str] = {}
135
136    for t in range(max_t):
137        gt_boxes_t = []
138        gt_ids_t = []
139        for tr in gt_tracks:
140            if t < tr.boxes.shape[0]:
141                gt_boxes_t.append(tr.boxes[t])
142                gt_ids_t.append(tr.track_id)
143
144        pred_boxes_t = []
145        pred_ids_t = []
146        for tr in pred_tracks:
147            if t < tr.boxes.shape[0]:
148                pred_boxes_t.append(tr.boxes[t])
149                pred_ids_t.append(tr.track_id)
150
151        total_gt += len(gt_boxes_t)
152
153        if len(pred_boxes_t) == 0:
154            total_fn += len(gt_boxes_t)
155            continue
156
157        # Greedy matching
158        matched_gt_idx: set[int] = set()
159        for _p_idx, (p_box, p_id) in enumerate(zip(pred_boxes_t, pred_ids_t, strict=False)):
160            best_iou, best_g = 0.0, -1
161            for g_idx, g_box in enumerate(gt_boxes_t):
162                if g_idx in matched_gt_idx:
163                    continue
164                iou = bbox_iou(np.array(p_box), np.array(g_box))
165                if iou > best_iou:
166                    best_iou, best_g = iou, g_idx
167            if best_iou >= iou_threshold and best_g >= 0:
168                matched_gt_idx.add(best_g)
169                g_id = gt_ids_t[best_g]
170                if p_id in pred_last_match and pred_last_match[p_id] != g_id:
171                    total_idsw += 1
172                pred_last_match[p_id] = g_id
173
174        total_fn += len(gt_boxes_t) - len(matched_gt_idx)
175        total_fp += len(pred_boxes_t) - len(matched_gt_idx)
176
177    mota = 1.0 - (total_fn + total_fp + total_idsw) / max(total_gt, 1)
178    mota = float(np.clip(mota, -1.0, 1.0))
179
180    # IDF1: identity recall × precision harmonic mean (simplified: treat matched as IDTP)
181    total_tp = total_gt - total_fn
182    idf1 = (2 * total_tp) / max(2 * total_tp + total_fp + total_fn, 1)
183    idf1 = float(np.clip(idf1, 0.0, 1.0))
184
185    return {
186        "mota": mota,
187        "idf1": idf1,
188        "fp": float(total_fp),
189        "fn": float(total_fn),
190        "idsw": float(total_idsw),
191    }
192
193
194def pose_metrics(
195    pred_rot: np.ndarray,
196    pred_trans: np.ndarray,
197    gt_rot: np.ndarray,
198    gt_trans: np.ndarray,
199) -> Dict[str, float]:
200    """Rotation geodesic error (degrees) and translation L2 error (meters)."""
201    pred_rot = np.asarray(pred_rot, dtype=np.float64)
202    gt_rot = np.asarray(gt_rot, dtype=np.float64)
203
204    # Handle quaternion input (4-vector)
205    if pred_rot.shape == (4,):
206        pred_rot = _quat_to_rotmat(pred_rot)
207    if gt_rot.shape == (4,):
208        gt_rot = _quat_to_rotmat(gt_rot)
209
210    # Geodesic distance: arccos((trace(R^T R') - 1) / 2)
211    relative = pred_rot.T @ gt_rot
212    trace_val = float(np.trace(relative))
213    cos_angle = (trace_val - 1.0) / 2.0
214    cos_angle = np.clip(cos_angle, -1.0, 1.0)
215    rot_err_deg = float(np.degrees(np.arccos(cos_angle)))
216
217    pred_t = np.asarray(pred_trans, dtype=np.float64)
218    gt_t = np.asarray(gt_trans, dtype=np.float64)
219
220    trans_err_m = float(np.linalg.norm(pred_t - gt_t))
221
222    # Translation angular error (scale-invariant direction error).
223    # Standard metric in ScanNet-1500 / MegaDepth-1500 / MapFree.
224    n_pred = np.linalg.norm(pred_t)
225    n_gt = np.linalg.norm(gt_t)
226    if n_pred < 1e-9 or n_gt < 1e-9:
227        trans_angular_deg = 180.0
228    else:
229        cos_a = np.clip(np.dot(pred_t, gt_t) / (n_pred * n_gt), -1.0, 1.0)
230        trans_angular_deg = float(np.degrees(np.arccos(cos_a)))
231
232    pose_error_max_deg = max(rot_err_deg, trans_angular_deg)
233
234    return {
235        "rotation_error_deg": rot_err_deg,
236        "translation_error_m": trans_err_m,
237        "translation_angular_deg": trans_angular_deg,
238        "pose_error_max_deg": pose_error_max_deg,
239    }
240
241
242def grounding_metrics(
243    pred_boxes: np.ndarray,
244    gt_boxes: np.ndarray,
245    iou_threshold: float = 0.5,
246) -> Dict[str, float]:
247    """Visual grounding: top-1 IoU and accuracy at IoU threshold.
248
249    Standard referring grounding protocol: a prediction is correct if the
250    highest-scoring predicted box has IoU ≥ threshold with any GT box.
251    """
252    if len(pred_boxes) == 0:
253        return {"grounding_iou": 0.0, "grounding_acc": 0.0}
254    if len(gt_boxes) == 0:
255        return {"grounding_iou": 1.0, "grounding_acc": 1.0}
256
257    # Evaluate the first (highest-scoring) prediction against all GT boxes
258    top_box = pred_boxes[0]
259    best_iou = max(bbox_iou(top_box, gt_box) for gt_box in gt_boxes)
260    acc = 1.0 if best_iou >= iou_threshold else 0.0
261
262    return {"grounding_iou": float(best_iou), "grounding_acc": float(acc)}
263
264
265def sparse_depth_metrics(
266    pred_coords: np.ndarray,
267    pred_depths: np.ndarray,
268    gt_coords: np.ndarray,
269    gt_depths: np.ndarray,
270    radius: float = 2.0,
271) -> Dict[str, float]:
272    """AbsRel and RMSE at sparse GT pixel locations.
273
274    We match predicted depths to GT coordinates by nearest-neighbor within
275    `radius` pixels.
276    """
277    if len(gt_coords) == 0:
278        return {"sparse_absrel": 0.0, "sparse_rmse": 0.0}
279
280    abs_errors = []
281    sq_errors = []
282    for (gx, gy), gd in zip(gt_coords, gt_depths, strict=False):
283        if len(pred_coords) == 0:
284            abs_errors.append(gd)
285            sq_errors.append(gd**2)
286            continue
287        dists = np.sqrt(((pred_coords - np.array([gx, gy])) ** 2).sum(axis=1))
288        nearest = int(np.argmin(dists))
289        if dists[nearest] <= radius:
290            pd = pred_depths[nearest]
291        else:
292            pd = 0.0  # no prediction at this location
293        abs_errors.append(abs(pd - gd) / max(gd, 1e-6))
294        sq_errors.append((pd - gd) ** 2)
295
296    return {
297        "sparse_absrel": float(np.mean(abs_errors)),
298        "sparse_rmse": float(np.sqrt(np.mean(sq_errors))),
299    }
300
301
302
303
304def keypoint_metrics(
305    pred_p0: np.ndarray,
306    pred_p1: np.ndarray,
307    gt_p0: np.ndarray,
308    gt_p1: np.ndarray,
309    visibility: np.ndarray | None = None,
310    px_threshold: float = 3.0,
311) -> Dict[str, float]:
312    """Keypoint matching accuracy: % correspondences within px_threshold pixels.
313
314    We match each GT keypoint pair to the nearest predicted pair in image0
315    and check if the predicted point in image1 is within threshold pixels of GT.
316    """
317    if len(gt_p0) == 0:
318        return {"keypoint_acc": 1.0, "mean_match_error": 0.0}
319    if len(pred_p0) == 0:
320        return {"keypoint_acc": 0.0, "mean_match_error": float("inf")}
321
322    mask = visibility if visibility is not None else np.ones(len(gt_p0), dtype=bool)
323    gt_p0_v = gt_p0[mask]
324    gt_p1_v = gt_p1[mask]
325
326    if len(gt_p0_v) == 0:
327        return {"keypoint_acc": 1.0, "mean_match_error": 0.0}
328
329    errors = []
330    for gp0, gp1 in zip(gt_p0_v, gt_p1_v, strict=False):
331        dists0 = np.sqrt(((pred_p0 - gp0) ** 2).sum(axis=1))
332        nearest = int(np.argmin(dists0))
333        err = float(np.sqrt(((pred_p1[nearest] - gp1) ** 2).sum()))
334        errors.append(err)
335
336    errors_arr = np.array(errors)
337    acc = float(np.mean(errors_arr < px_threshold))
338    mean_err = float(np.mean(errors_arr))
339
340    return {"keypoint_acc": acc, "mean_match_error": mean_err}
341
342
343# ------------------------------------------------------------------ #
344# Geometry helpers
345# ------------------------------------------------------------------ #
346
347
348def bbox_iou(box_a: np.ndarray, box_b: np.ndarray) -> float:
349    ax1, ay1, ax2, ay2 = box_a
350    bx1, by1, bx2, by2 = box_b
351    inter_x1, inter_y1 = max(ax1, bx1), max(ay1, by1)
352    inter_x2, inter_y2 = min(ax2, bx2), min(ay2, by2)
353    inter_w = max(0.0, inter_x2 - inter_x1)
354    inter_h = max(0.0, inter_y2 - inter_y1)
355    inter_area = inter_w * inter_h
356    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)
357    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)
358    union = area_a + area_b - inter_area
359    return float(inter_area / union) if union > 0 else 0.0
360
361
362def _quat_to_rotmat(q: np.ndarray) -> np.ndarray:
363    """Convert quaternion [w, x, y, z] to 3x3 rotation matrix."""
364    w, x, y, z = q / np.linalg.norm(q)
365    return np.array(
366        [
367            [1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w],
368            [2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w],
369            [2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y],
370        ],
371        dtype=np.float64,
372    )