rpx_benchmark.vqa.metrics
Benchmark metrics and smoke-gate aggregation for RPX VQA.
1"""Benchmark metrics and smoke-gate aggregation for RPX VQA.""" 2 3from __future__ import annotations 4 5import math 6from collections import defaultdict 7from typing import Any, Iterable 8 9from ..exceptions import MetricError 10from .contract import ATTRIBUTE_TYPES, BBOX_TYPES, BINARY_TYPES, VQASample 11from .outputs import ParsedOutput, normalize_label 12 13COCO_IOU_THRESHOLDS = tuple(value / 100 for value in range(50, 100, 5)) 14 15 16def bbox_iou( 17 predicted: tuple[float, float, float, float], ground_truth: tuple[int, int, int, int] 18) -> float: 19 """IoU for the dataset's inclusive pixel xyxy convention.""" 20 px0, py0, px1, py1 = predicted 21 gx0, gy0, gx1, gy1 = ground_truth 22 ix0, iy0, ix1, iy1 = max(px0, gx0), max(py0, gy0), min(px1, gx1), min(py1, gy1) 23 intersection = max(0.0, ix1 - ix0 + 1) * max(0.0, iy1 - iy0 + 1) 24 pred_area = max(0.0, px1 - px0 + 1) * max(0.0, py1 - py0 + 1) 25 gt_area = (gx1 - gx0 + 1) * (gy1 - gy0 + 1) 26 union = pred_area + gt_area - intersection 27 return intersection / union if union else 0.0 28 29 30def bbox_is_valid( 31 predicted: tuple[float, float, float, float] | None, 32 image_width: int, 33 image_height: int, 34) -> bool: 35 """Whether an inclusive-pixel XYXY prediction is finite and in bounds.""" 36 if predicted is None or image_width < 1 or image_height < 1: 37 return False 38 x0, y0, x1, y1 = predicted 39 return ( 40 all(math.isfinite(value) for value in predicted) 41 and 0 <= x0 <= x1 < image_width 42 and 0 <= y0 <= y1 < image_height 43 ) 44 45 46def bbox_generalized_iou( 47 predicted: tuple[float, float, float, float], 48 ground_truth: tuple[int, int, int, int], 49) -> float: 50 """Generalized IoU for the dataset's inclusive-pixel XYXY convention.""" 51 px0, py0, px1, py1 = predicted 52 gx0, gy0, gx1, gy1 = ground_truth 53 ix0, iy0, ix1, iy1 = max(px0, gx0), max(py0, gy0), min(px1, gx1), min(py1, gy1) 54 intersection = max(0.0, ix1 - ix0 + 1) * max(0.0, iy1 - iy0 + 1) 55 pred_area = max(0.0, px1 - px0 + 1) * max(0.0, py1 - py0 + 1) 56 gt_area = max(0.0, gx1 - gx0 + 1) * max(0.0, gy1 - gy0 + 1) 57 union = pred_area + gt_area - intersection 58 iou = intersection / union if union else 0.0 59 enclosing_area = ( 60 max(0.0, max(px1, gx1) - min(px0, gx0) + 1) 61 * max(0.0, max(py1, gy1) - min(py0, gy0) + 1) 62 ) 63 return iou - (enclosing_area - union) / enclosing_area if enclosing_area else -1.0 64 65 66def bbox_center_in_ground_truth( 67 predicted: tuple[float, float, float, float], 68 ground_truth: tuple[int, int, int, int], 69) -> bool: 70 """Whether the predicted-box center lies inside the inclusive GT box.""" 71 px0, py0, px1, py1 = predicted 72 gx0, gy0, gx1, gy1 = ground_truth 73 center_x = (px0 + px1) / 2 74 center_y = (py0 + py1) / 2 75 return gx0 <= center_x <= gx1 and gy0 <= center_y <= gy1 76 77 78def _mean(values: list[float]) -> float: 79 return sum(values) / len(values) if values else 0.0 80 81 82def label_token_f1(predicted: str | None, expected: str) -> float: 83 """Transparent lexical overlap for the single-reference label contract. 84 85 This complements, but never replaces, normalized exact match. It is not 86 called semantic accuracy: RPX has one reference label per row rather than 87 the multiple human answers needed by the conventional VQA soft score. 88 """ 89 predicted_tokens = normalize_label(predicted or "").split() 90 expected_tokens = normalize_label(expected).split() 91 if not predicted_tokens or not expected_tokens: 92 return 0.0 93 predicted_counts: dict[str, int] = defaultdict(int) 94 expected_counts: dict[str, int] = defaultdict(int) 95 for token in predicted_tokens: 96 predicted_counts[token] += 1 97 for token in expected_tokens: 98 expected_counts[token] += 1 99 overlap = sum( 100 min(count, expected_counts.get(token, 0)) 101 for token, count in predicted_counts.items() 102 ) 103 if overlap == 0: 104 return 0.0 105 precision = overlap / len(predicted_tokens) 106 recall = overlap / len(expected_tokens) 107 return 2 * precision * recall / (precision + recall) 108 109 110def score_predictions( 111 pairs: Iterable[tuple[VQASample, ParsedOutput]], iou_threshold: float = 0.5 112) -> dict[str, Any]: 113 rows = list(pairs) 114 if not rows: 115 raise MetricError("no VQA predictions to score", hint="provide one output per request") 116 valid = [float(parsed.valid) for _, parsed in rows] 117 by_type: dict[str, list[float]] = defaultdict(list) 118 binary: list[float] = [] 119 attribute: list[float] = [] 120 ious: list[float] = [] 121 bbox_hits: list[float] = [] 122 bbox_labels: list[float] = [] 123 bbox_label_token_f1: list[float] = [] 124 bbox_joint_hits: list[float] = [] 125 bbox_validity: list[float] = [] 126 bbox_gious: list[float] = [] 127 bbox_valid_gious: list[float] = [] 128 bbox_center_hits: list[float] = [] 129 bbox_hits_by_threshold: dict[float, list[float]] = { 130 threshold: [] for threshold in (0.25, 0.5, 0.75, *COCO_IOU_THRESHOLDS) 131 } 132 detailed_by_type: dict[str, list[dict[str, float]]] = defaultdict(list) 133 for sample, parsed in rows: 134 score = 0.0 135 if parsed.valid and sample.question_type in BINARY_TYPES: 136 score = float(parsed.label == sample.answer) 137 binary.append(score) 138 elif parsed.valid and sample.question_type in ATTRIBUTE_TYPES: 139 score = float(parsed.label == normalize_label(sample.answer)) 140 attribute.append(score) 141 elif sample.question_type in BBOX_TYPES: 142 assert sample.answer_bbox is not None 143 bbox_valid = bool( 144 parsed.valid 145 and bbox_is_valid(parsed.bbox, sample.img_w, sample.img_h) 146 ) 147 iou = 0.0 148 giou = -1.0 149 center_hit = 0.0 150 if bbox_valid and parsed.bbox is not None: 151 iou = bbox_iou(parsed.bbox, sample.answer_bbox) 152 giou = bbox_generalized_iou(parsed.bbox, sample.answer_bbox) 153 center_hit = float( 154 bbox_center_in_ground_truth(parsed.bbox, sample.answer_bbox) 155 ) 156 ious.append(iou) 157 bbox_validity.append(float(bbox_valid)) 158 bbox_gious.append(giou) 159 if bbox_valid: 160 bbox_valid_gious.append(giou) 161 bbox_center_hits.append(center_hit) 162 score = float(iou >= iou_threshold) 163 bbox_hits.append(score) 164 label_exact = float(parsed.label == normalize_label(sample.answer)) 165 label_f1 = label_token_f1(parsed.label, sample.answer) 166 # Missing/unparseable labels count as zero; excluding them would 167 # inflate textual correctness precisely when protocol compliance 168 # is poor. 169 bbox_labels.append(label_exact) 170 bbox_label_token_f1.append(label_f1) 171 bbox_joint_hits.append(float(label_exact == 1.0 and iou >= iou_threshold)) 172 for threshold, values in bbox_hits_by_threshold.items(): 173 values.append(float(iou >= threshold)) 174 detailed_by_type[sample.question_type].append( 175 { 176 "valid": float(parsed.valid), 177 "bbox_valid": float(bbox_valid), 178 "iou": iou, 179 "giou": giou, 180 "center_in_gt": center_hit, 181 "label_exact": label_exact, 182 "label_token_f1": label_f1, 183 } 184 ) 185 by_type[sample.question_type].append(score) 186 per_type_detailed = {} 187 for question_type, detail_rows in sorted(detailed_by_type.items()): 188 per_type_detailed[question_type] = { 189 "count": len(detail_rows), 190 "parse_rate": _mean([value["valid"] for value in detail_rows]), 191 "bbox_validity_rate": _mean([value["bbox_valid"] for value in detail_rows]), 192 "bbox_mean_iou": _mean([value["iou"] for value in detail_rows]), 193 "bbox_mean_giou": _mean([value["giou"] for value in detail_rows]), 194 "bbox_mean_giou_valid": _mean( 195 [value["giou"] for value in detail_rows if value["bbox_valid"]] 196 ), 197 "bbox_center_in_gt_accuracy": _mean( 198 [value["center_in_gt"] for value in detail_rows] 199 ), 200 "bbox_accuracy_at_0_25": _mean( 201 [float(value["iou"] >= 0.25) for value in detail_rows] 202 ), 203 "bbox_accuracy_at_0_5": _mean( 204 [float(value["iou"] >= 0.5) for value in detail_rows] 205 ), 206 "bbox_accuracy_at_0_75": _mean( 207 [float(value["iou"] >= 0.75) for value in detail_rows] 208 ), 209 "label_normalized_exact_match": _mean( 210 [value["label_exact"] for value in detail_rows] 211 ), 212 "label_token_f1": _mean([value["label_token_f1"] for value in detail_rows]), 213 } 214 return { 215 "count": len(rows), 216 "primary_metric": "bbox_accuracy_at_0_5", 217 "bbox_scoring_uses_generated_label": False, 218 "parse_rate": _mean(valid), 219 "binary_accuracy": _mean(binary), 220 "attribute_exact_match": _mean(attribute), 221 "bbox_count": len(ious), 222 "bbox_validity_rate": _mean(bbox_validity), 223 "bbox_accuracy_at_0_5": _mean(bbox_hits), 224 "bbox_accuracy_at_0_25": _mean(bbox_hits_by_threshold[0.25]), 225 "bbox_accuracy_at_0_75": _mean(bbox_hits_by_threshold[0.75]), 226 "bbox_mean_iou": _mean(ious), 227 "bbox_mean_giou": _mean(bbox_gious), 228 "bbox_mean_giou_valid": _mean(bbox_valid_gious), 229 "bbox_center_in_gt_accuracy": _mean(bbox_center_hits), 230 "bbox_mean_accuracy_50_95": _mean( 231 [_mean(bbox_hits_by_threshold[threshold]) for threshold in COCO_IOU_THRESHOLDS] 232 ), 233 # Retain the old key for report compatibility, but make its exact 234 # single-reference meaning explicit in the new key. 235 "bbox_label_accuracy": _mean(bbox_labels), 236 "bbox_label_normalized_exact_match": _mean(bbox_labels), 237 "bbox_label_token_f1": _mean(bbox_label_token_f1), 238 "bbox_joint_label_exact_and_iou_at_0_5": _mean(bbox_joint_hits), 239 "per_type": {key: _mean(value) for key, value in sorted(by_type.items())}, 240 "per_type_detailed": per_type_detailed, 241 "metric_notes": { 242 "bbox_validity_rate": ( 243 "Fraction of bbox-task rows with a parsed, finite, ordered inclusive-XYXY " 244 "box wholly inside the target image." 245 ), 246 "bbox_mean_giou": ( 247 "Mean generalized IoU over every bbox-task row; an invalid or missing box " 248 "receives -1.0, the worst possible GIoU, to prevent parse failures from " 249 "inflating localization quality." 250 ), 251 "bbox_mean_giou_valid": ( 252 "Diagnostic mean GIoU over valid boxes only; use bbox_mean_giou for the " 253 "all-row model comparison." 254 ), 255 "bbox_center_in_gt_accuracy": ( 256 "Fraction of every bbox-task row whose predicted-box center lies inside " 257 "the inclusive ground-truth box; invalid or missing boxes count as misses." 258 ), 259 "bbox_mean_accuracy_50_95": ( 260 "Mean single-box success rate over IoU thresholds 0.50:0.05:0.95; " 261 "not detection mAP because predictions have no confidence scores." 262 ), 263 "bbox_label_normalized_exact_match": ( 264 "Informational only: one-reference normalized exact match; bbox scoring " 265 "does not depend on the generated label." 266 ), 267 "bbox_label_token_f1": ( 268 "Informational only: lexical token overlap; bbox scoring does not depend " 269 "on the generated label." 270 ), 271 "bbox_joint_label_exact_and_iou_at_0_5": ( 272 "Informational diagnostic only; the benchmark's bbox result is " 273 "bbox_accuracy_at_0_5." 274 ), 275 }, 276 }
COCO_IOU_THRESHOLDS =
(0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95)
def
bbox_iou( predicted: tuple[float, float, float, float], ground_truth: tuple[int, int, int, int]) -> float:
17def bbox_iou( 18 predicted: tuple[float, float, float, float], ground_truth: tuple[int, int, int, int] 19) -> float: 20 """IoU for the dataset's inclusive pixel xyxy convention.""" 21 px0, py0, px1, py1 = predicted 22 gx0, gy0, gx1, gy1 = ground_truth 23 ix0, iy0, ix1, iy1 = max(px0, gx0), max(py0, gy0), min(px1, gx1), min(py1, gy1) 24 intersection = max(0.0, ix1 - ix0 + 1) * max(0.0, iy1 - iy0 + 1) 25 pred_area = max(0.0, px1 - px0 + 1) * max(0.0, py1 - py0 + 1) 26 gt_area = (gx1 - gx0 + 1) * (gy1 - gy0 + 1) 27 union = pred_area + gt_area - intersection 28 return intersection / union if union else 0.0
IoU for the dataset's inclusive pixel xyxy convention.
def
bbox_is_valid( predicted: tuple[float, float, float, float] | None, image_width: int, image_height: int) -> bool:
31def bbox_is_valid( 32 predicted: tuple[float, float, float, float] | None, 33 image_width: int, 34 image_height: int, 35) -> bool: 36 """Whether an inclusive-pixel XYXY prediction is finite and in bounds.""" 37 if predicted is None or image_width < 1 or image_height < 1: 38 return False 39 x0, y0, x1, y1 = predicted 40 return ( 41 all(math.isfinite(value) for value in predicted) 42 and 0 <= x0 <= x1 < image_width 43 and 0 <= y0 <= y1 < image_height 44 )
Whether an inclusive-pixel XYXY prediction is finite and in bounds.
def
bbox_generalized_iou( predicted: tuple[float, float, float, float], ground_truth: tuple[int, int, int, int]) -> float:
47def bbox_generalized_iou( 48 predicted: tuple[float, float, float, float], 49 ground_truth: tuple[int, int, int, int], 50) -> float: 51 """Generalized IoU for the dataset's inclusive-pixel XYXY convention.""" 52 px0, py0, px1, py1 = predicted 53 gx0, gy0, gx1, gy1 = ground_truth 54 ix0, iy0, ix1, iy1 = max(px0, gx0), max(py0, gy0), min(px1, gx1), min(py1, gy1) 55 intersection = max(0.0, ix1 - ix0 + 1) * max(0.0, iy1 - iy0 + 1) 56 pred_area = max(0.0, px1 - px0 + 1) * max(0.0, py1 - py0 + 1) 57 gt_area = max(0.0, gx1 - gx0 + 1) * max(0.0, gy1 - gy0 + 1) 58 union = pred_area + gt_area - intersection 59 iou = intersection / union if union else 0.0 60 enclosing_area = ( 61 max(0.0, max(px1, gx1) - min(px0, gx0) + 1) 62 * max(0.0, max(py1, gy1) - min(py0, gy0) + 1) 63 ) 64 return iou - (enclosing_area - union) / enclosing_area if enclosing_area else -1.0
Generalized IoU for the dataset's inclusive-pixel XYXY convention.
def
bbox_center_in_ground_truth( predicted: tuple[float, float, float, float], ground_truth: tuple[int, int, int, int]) -> bool:
67def bbox_center_in_ground_truth( 68 predicted: tuple[float, float, float, float], 69 ground_truth: tuple[int, int, int, int], 70) -> bool: 71 """Whether the predicted-box center lies inside the inclusive GT box.""" 72 px0, py0, px1, py1 = predicted 73 gx0, gy0, gx1, gy1 = ground_truth 74 center_x = (px0 + px1) / 2 75 center_y = (py0 + py1) / 2 76 return gx0 <= center_x <= gx1 and gy0 <= center_y <= gy1
Whether the predicted-box center lies inside the inclusive GT box.
def
label_token_f1(predicted: str | None, expected: str) -> float:
83def label_token_f1(predicted: str | None, expected: str) -> float: 84 """Transparent lexical overlap for the single-reference label contract. 85 86 This complements, but never replaces, normalized exact match. It is not 87 called semantic accuracy: RPX has one reference label per row rather than 88 the multiple human answers needed by the conventional VQA soft score. 89 """ 90 predicted_tokens = normalize_label(predicted or "").split() 91 expected_tokens = normalize_label(expected).split() 92 if not predicted_tokens or not expected_tokens: 93 return 0.0 94 predicted_counts: dict[str, int] = defaultdict(int) 95 expected_counts: dict[str, int] = defaultdict(int) 96 for token in predicted_tokens: 97 predicted_counts[token] += 1 98 for token in expected_tokens: 99 expected_counts[token] += 1 100 overlap = sum( 101 min(count, expected_counts.get(token, 0)) 102 for token, count in predicted_counts.items() 103 ) 104 if overlap == 0: 105 return 0.0 106 precision = overlap / len(predicted_tokens) 107 recall = overlap / len(expected_tokens) 108 return 2 * precision * recall / (precision + recall)
Transparent lexical overlap for the single-reference label contract.
This complements, but never replaces, normalized exact match. It is not called semantic accuracy: RPX has one reference label per row rather than the multiple human answers needed by the conventional VQA soft score.
def
score_predictions( pairs: Iterable[tuple[rpx_benchmark.vqa.VQASample, rpx_benchmark.vqa.ParsedOutput]], iou_threshold: float = 0.5) -> dict[str, typing.Any]:
111def score_predictions( 112 pairs: Iterable[tuple[VQASample, ParsedOutput]], iou_threshold: float = 0.5 113) -> dict[str, Any]: 114 rows = list(pairs) 115 if not rows: 116 raise MetricError("no VQA predictions to score", hint="provide one output per request") 117 valid = [float(parsed.valid) for _, parsed in rows] 118 by_type: dict[str, list[float]] = defaultdict(list) 119 binary: list[float] = [] 120 attribute: list[float] = [] 121 ious: list[float] = [] 122 bbox_hits: list[float] = [] 123 bbox_labels: list[float] = [] 124 bbox_label_token_f1: list[float] = [] 125 bbox_joint_hits: list[float] = [] 126 bbox_validity: list[float] = [] 127 bbox_gious: list[float] = [] 128 bbox_valid_gious: list[float] = [] 129 bbox_center_hits: list[float] = [] 130 bbox_hits_by_threshold: dict[float, list[float]] = { 131 threshold: [] for threshold in (0.25, 0.5, 0.75, *COCO_IOU_THRESHOLDS) 132 } 133 detailed_by_type: dict[str, list[dict[str, float]]] = defaultdict(list) 134 for sample, parsed in rows: 135 score = 0.0 136 if parsed.valid and sample.question_type in BINARY_TYPES: 137 score = float(parsed.label == sample.answer) 138 binary.append(score) 139 elif parsed.valid and sample.question_type in ATTRIBUTE_TYPES: 140 score = float(parsed.label == normalize_label(sample.answer)) 141 attribute.append(score) 142 elif sample.question_type in BBOX_TYPES: 143 assert sample.answer_bbox is not None 144 bbox_valid = bool( 145 parsed.valid 146 and bbox_is_valid(parsed.bbox, sample.img_w, sample.img_h) 147 ) 148 iou = 0.0 149 giou = -1.0 150 center_hit = 0.0 151 if bbox_valid and parsed.bbox is not None: 152 iou = bbox_iou(parsed.bbox, sample.answer_bbox) 153 giou = bbox_generalized_iou(parsed.bbox, sample.answer_bbox) 154 center_hit = float( 155 bbox_center_in_ground_truth(parsed.bbox, sample.answer_bbox) 156 ) 157 ious.append(iou) 158 bbox_validity.append(float(bbox_valid)) 159 bbox_gious.append(giou) 160 if bbox_valid: 161 bbox_valid_gious.append(giou) 162 bbox_center_hits.append(center_hit) 163 score = float(iou >= iou_threshold) 164 bbox_hits.append(score) 165 label_exact = float(parsed.label == normalize_label(sample.answer)) 166 label_f1 = label_token_f1(parsed.label, sample.answer) 167 # Missing/unparseable labels count as zero; excluding them would 168 # inflate textual correctness precisely when protocol compliance 169 # is poor. 170 bbox_labels.append(label_exact) 171 bbox_label_token_f1.append(label_f1) 172 bbox_joint_hits.append(float(label_exact == 1.0 and iou >= iou_threshold)) 173 for threshold, values in bbox_hits_by_threshold.items(): 174 values.append(float(iou >= threshold)) 175 detailed_by_type[sample.question_type].append( 176 { 177 "valid": float(parsed.valid), 178 "bbox_valid": float(bbox_valid), 179 "iou": iou, 180 "giou": giou, 181 "center_in_gt": center_hit, 182 "label_exact": label_exact, 183 "label_token_f1": label_f1, 184 } 185 ) 186 by_type[sample.question_type].append(score) 187 per_type_detailed = {} 188 for question_type, detail_rows in sorted(detailed_by_type.items()): 189 per_type_detailed[question_type] = { 190 "count": len(detail_rows), 191 "parse_rate": _mean([value["valid"] for value in detail_rows]), 192 "bbox_validity_rate": _mean([value["bbox_valid"] for value in detail_rows]), 193 "bbox_mean_iou": _mean([value["iou"] for value in detail_rows]), 194 "bbox_mean_giou": _mean([value["giou"] for value in detail_rows]), 195 "bbox_mean_giou_valid": _mean( 196 [value["giou"] for value in detail_rows if value["bbox_valid"]] 197 ), 198 "bbox_center_in_gt_accuracy": _mean( 199 [value["center_in_gt"] for value in detail_rows] 200 ), 201 "bbox_accuracy_at_0_25": _mean( 202 [float(value["iou"] >= 0.25) for value in detail_rows] 203 ), 204 "bbox_accuracy_at_0_5": _mean( 205 [float(value["iou"] >= 0.5) for value in detail_rows] 206 ), 207 "bbox_accuracy_at_0_75": _mean( 208 [float(value["iou"] >= 0.75) for value in detail_rows] 209 ), 210 "label_normalized_exact_match": _mean( 211 [value["label_exact"] for value in detail_rows] 212 ), 213 "label_token_f1": _mean([value["label_token_f1"] for value in detail_rows]), 214 } 215 return { 216 "count": len(rows), 217 "primary_metric": "bbox_accuracy_at_0_5", 218 "bbox_scoring_uses_generated_label": False, 219 "parse_rate": _mean(valid), 220 "binary_accuracy": _mean(binary), 221 "attribute_exact_match": _mean(attribute), 222 "bbox_count": len(ious), 223 "bbox_validity_rate": _mean(bbox_validity), 224 "bbox_accuracy_at_0_5": _mean(bbox_hits), 225 "bbox_accuracy_at_0_25": _mean(bbox_hits_by_threshold[0.25]), 226 "bbox_accuracy_at_0_75": _mean(bbox_hits_by_threshold[0.75]), 227 "bbox_mean_iou": _mean(ious), 228 "bbox_mean_giou": _mean(bbox_gious), 229 "bbox_mean_giou_valid": _mean(bbox_valid_gious), 230 "bbox_center_in_gt_accuracy": _mean(bbox_center_hits), 231 "bbox_mean_accuracy_50_95": _mean( 232 [_mean(bbox_hits_by_threshold[threshold]) for threshold in COCO_IOU_THRESHOLDS] 233 ), 234 # Retain the old key for report compatibility, but make its exact 235 # single-reference meaning explicit in the new key. 236 "bbox_label_accuracy": _mean(bbox_labels), 237 "bbox_label_normalized_exact_match": _mean(bbox_labels), 238 "bbox_label_token_f1": _mean(bbox_label_token_f1), 239 "bbox_joint_label_exact_and_iou_at_0_5": _mean(bbox_joint_hits), 240 "per_type": {key: _mean(value) for key, value in sorted(by_type.items())}, 241 "per_type_detailed": per_type_detailed, 242 "metric_notes": { 243 "bbox_validity_rate": ( 244 "Fraction of bbox-task rows with a parsed, finite, ordered inclusive-XYXY " 245 "box wholly inside the target image." 246 ), 247 "bbox_mean_giou": ( 248 "Mean generalized IoU over every bbox-task row; an invalid or missing box " 249 "receives -1.0, the worst possible GIoU, to prevent parse failures from " 250 "inflating localization quality." 251 ), 252 "bbox_mean_giou_valid": ( 253 "Diagnostic mean GIoU over valid boxes only; use bbox_mean_giou for the " 254 "all-row model comparison." 255 ), 256 "bbox_center_in_gt_accuracy": ( 257 "Fraction of every bbox-task row whose predicted-box center lies inside " 258 "the inclusive ground-truth box; invalid or missing boxes count as misses." 259 ), 260 "bbox_mean_accuracy_50_95": ( 261 "Mean single-box success rate over IoU thresholds 0.50:0.05:0.95; " 262 "not detection mAP because predictions have no confidence scores." 263 ), 264 "bbox_label_normalized_exact_match": ( 265 "Informational only: one-reference normalized exact match; bbox scoring " 266 "does not depend on the generated label." 267 ), 268 "bbox_label_token_f1": ( 269 "Informational only: lexical token overlap; bbox scoring does not depend " 270 "on the generated label." 271 ), 272 "bbox_joint_label_exact_and_iou_at_0_5": ( 273 "Informational diagnostic only; the benchmark's bbox result is " 274 "bbox_accuracy_at_0_5." 275 ), 276 }, 277 }