rpx_benchmark.vqa

11@dataclass(frozen=True)
12class ModelSpec:
13    key: str
14    display_name: str
15    size_b: float
16    vram_gb: int
17    bbox_mode: str
18    capabilities: frozenset[str]
19    diagnostic_capabilities: frozenset[str] = frozenset()

36@dataclass(frozen=True)
37class ParsedOutput:
38    valid: bool
39    label: str | None = None
40    bbox: tuple[float, float, float, float] | None = None
41    error: str | None = None
42    coordinate_format: str | None = None

12@dataclass(frozen=True)
13class PromptSpec:
14    text: str
15    max_new_tokens: int
16    output_kind: str

118@dataclass(frozen=True)
119class VQASample:
120    sample_id: str
121    scene_id: str
122    kind: str
123    phase: int | None
124    frame: str
125    image: dict[str, str]
126    img_w: int
127    img_h: int
128    question_type: str
129    question: str
130    answer: str
131    answer_bbox: tuple[int, int, int, int] | None = None
132    evidence: dict[str, Any] | None = None
133    schema_version: str = SCHEMA_VERSION
134    # In-context (two-image) fields. None on every normal row.
135    reference_image: dict[str, str] | None = None
136    reference_crop_bbox: tuple[int, int, int, int] | None = None
137    reference_crop_sha256: str | None = None
138    source_sample_id: str | None = None
139
140    @property
141    def is_in_context(self) -> bool:
142        return self.question_type in INCONTEXT_BBOX_TYPES
143
144    @property
145    def images(self) -> tuple[dict[str, str], ...]:
146        """Ordered locators to send to a model: [reference, target] for
147        in-context samples, [target] alone for normal samples."""
148        if self.is_in_context:
149            assert self.reference_image is not None
150            return (self.reference_image, self.image)
151        return (self.image,)
152
153    @classmethod
154    def from_dict(cls, row: dict[str, Any]) -> "VQASample":
155        bbox = row.get("answer_bbox")
156        evidence = row.get("evidence")
157        if isinstance(evidence, str):
158            evidence = json.loads(evidence)
159        question_type = str(row.get("question_type") or row["type"])
160        in_context = question_type in INCONTEXT_BBOX_TYPES
161        # In-context Parquet rows store their target/reference locators under
162        # target_image/reference_image; a round-tripped manifest row (written
163        # by write_manifest, reloaded by load_manifest) stores them under the
164        # canonical image/reference_image keys. Accept either spelling.
165        target_locator = row.get("image") or row.get("target_image")
166        if target_locator is None:
167            target_locator = image_locator(row)
168        target_locator = {key: str(target_locator[key]) for key in _REQUIRED_LOCATOR_KEYS}
169        reference_locator = row.get("reference_image")
170        reference_crop_bbox = row.get("reference_crop_bbox")
171        if reference_locator is not None:
172            if reference_crop_bbox is None and "crop_bbox" in reference_locator:
173                # The raw in-context Parquet's reference_image struct also
174                # carries a redundant crop_bbox key; prefer the top-level
175                # reference_crop_bbox column when present (it always is in
176                # the current release), falling back to the nested copy.
177                reference_crop_bbox = reference_locator["crop_bbox"]
178            reference_locator = {key: str(reference_locator[key]) for key in _REQUIRED_LOCATOR_KEYS}
179        raw_phase = row.get("phase")
180        is_null_phase = raw_phase is None or (isinstance(raw_phase, float) and raw_phase != raw_phase)
181        sample = cls(
182            sample_id=str(row.get("sample_id") or stable_sample_id(row)),
183            scene_id=str(row["scene_id"]),
184            kind=str(row["kind"]),
185            phase=None if raw_phase is None or is_null_phase else int(raw_phase),
186            frame=str(row["frame"]),
187            image=dict(target_locator),
188            img_w=int(row["img_w"]),
189            img_h=int(row["img_h"]),
190            question_type=question_type,
191            question=str(row["question"]),
192            answer=str(row["answer"]),
193            answer_bbox=None if bbox is None else _bbox_tuple(bbox),
194            evidence=evidence,
195            schema_version=str(row.get("schema_version", SCHEMA_VERSION)),
196            reference_image=dict(reference_locator) if reference_locator is not None else None,
197            reference_crop_bbox=(
198                None if reference_crop_bbox is None else _bbox_tuple(reference_crop_bbox)
199            ),
200            reference_crop_sha256=(
201                None
202                if row.get("reference_crop_sha256") is None
203                else str(row["reference_crop_sha256"])
204            ),
205            source_sample_id=(
206                None if row.get("source_sample_id") is None else str(row["source_sample_id"])
207            ),
208        )
209        if in_context and reference_locator is None:
210            raise ManifestError(
211                f"in-context VQA type {question_type!r} is missing reference_image",
212                hint="regenerate from the in-context Parquet, not the single-image one",
213            )
214        sample.validate()
215        return sample
216
217    def validate(self) -> None:
218        if self.schema_version not in {SCHEMA_VERSION, "incontext_v1"}:
219            raise ManifestError(
220                f"unsupported VQA schema_version: {self.schema_version}",
221                hint=f"regenerate with schema {SCHEMA_VERSION} or incontext_v1",
222            )
223        if self.question_type not in TASK_TYPES:
224            raise ManifestError(
225                f"out-of-scope VQA type: {self.question_type}", hint="filter to the frozen task set"
226            )
227        if self.kind not in {"mos", "ego"}:
228            raise ManifestError(f"invalid VQA kind: {self.kind}", hint="use 'mos' or 'ego'")
229        if self.kind == "mos" and self.phase not in {0, 1, 2}:
230            raise ManifestError("MOS sample has an invalid phase", hint="use phase 0, 1, or 2")
231        if self.kind == "ego" and self.phase is not None:
232            raise ManifestError("ego sample has a phase", hint="set phase to null for ego")
233        if self.question_type in BINARY_TYPES and self.answer not in {"yes", "no"}:
234            raise ManifestError("invalid binary answer", hint="use exactly 'yes' or 'no'")
235        if self.question_type in BBOX_TYPES and self.answer_bbox is None:
236            raise ManifestError("bbox task is missing answer_bbox", hint="regenerate its ground truth")
237        if self.answer_bbox is not None:
238            # Always checked against the TARGET image's own img_w/img_h --
239            # for in-context rows that is Image 2, never Image 1 (reference).
240            x0, y0, x1, y1 = self.answer_bbox
241            if not (0 <= x0 <= x1 < self.img_w and 0 <= y0 <= y1 < self.img_h):
242                raise ManifestError(
243                    f"invalid inclusive xyxy bbox: {self.answer_bbox}",
244                    hint="keep all coordinates ordered and inside the original image",
245                )
246        if self.is_in_context:
247            self._validate_in_context_locators()
248        else:
249            expected = image_locator(
250                {
251                    "scene_id": self.scene_id,
252                    "kind": self.kind,
253                    "phase": self.phase,
254                    "frame": self.frame,
255                }
256            )
257            if self.image != expected:
258                raise ManifestError(
259                    f"non-canonical VQA image locator: {self.image!r}",
260                    hint=f"use {expected!r}",
261                )
262
263    def _validate_in_context_locators(self) -> None:
264        # In-context locators are taken verbatim from the in-context Parquet
265        # (see module docstring) -- never recomputed via image_locator. What
266        # IS enforced: both locators are structurally complete and pinned to
267        # an immutable (non-"main"/non-branch-name) revision, and the
268        # reference crop has the bbox+hash needed to reconstruct Image 1.
269        if self.reference_image is None:
270            raise ManifestError(
271                "in-context sample is missing reference_image", hint="regenerate from the in-context Parquet"
272            )
273        for label, locator in (("target_image", self.image), ("reference_image", self.reference_image)):
274            missing = _REQUIRED_LOCATOR_KEYS - locator.keys()
275            if missing:
276                raise ManifestError(
277                    f"{label} locator is missing keys: {sorted(missing)}",
278                    hint=f"expected keys {sorted(_REQUIRED_LOCATOR_KEYS)}",
279                )
280            revision = str(locator["revision"])
281            if revision in {"main", "master"} or not re.fullmatch(r"[0-9a-f]{40}", revision):
282                raise ManifestError(
283                    f"{label} is not pinned to an immutable revision: {revision!r}",
284                    hint="use the exact revision stored in the in-context Parquet",
285                )
286        if self.reference_crop_bbox is None:
287            raise ManifestError(
288                "in-context sample is missing reference_crop_bbox",
289                hint="regenerate from the in-context Parquet",
290            )
291        if self.reference_crop_sha256 is None or not re.fullmatch(
292            r"[0-9a-f]{64}", self.reference_crop_sha256
293        ):
294            raise ManifestError(
295                f"invalid reference_crop_sha256: {self.reference_crop_sha256!r}",
296                hint="expected a 64-char lowercase hex sha256",
297            )
298
299    def to_dict(self) -> dict[str, Any]:
300        return {
301            "schema_version": self.schema_version,
302            "sample_id": self.sample_id,
303            "scene_id": self.scene_id,
304            "kind": self.kind,
305            "phase": self.phase,
306            "frame": self.frame,
307            "image": self.image,
308            "img_w": self.img_w,
309            "img_h": self.img_h,
310            "question_type": self.question_type,
311            "question": self.question,
312            "answer": self.answer,
313            "answer_bbox": list(self.answer_bbox) if self.answer_bbox is not None else None,
314            "bbox_format": "xyxy_pixel_inclusive" if self.answer_bbox is not None else None,
315            "evidence": self.evidence,
316            "reference_image": self.reference_image,
317            "reference_crop_bbox": (
318                list(self.reference_crop_bbox) if self.reference_crop_bbox is not None else None
319            ),
320            "reference_crop_sha256": self.reference_crop_sha256,
321            "source_sample_id": self.source_sample_id,
322        }

20def evaluate_vqa(
21    manifest_path: str | Path,
22    predict: Callable[[tuple[Path, ...], str, int, str], str],
23    *,
24    model_name: str,
25    output_dir: str | Path,
26    image_cache: str | Path,
27    model_revision: str,
28) -> dict:
29    """Evaluate a local model or API callable, without changing the model roster.
30
31    ``predict(image_paths, prompt, max_new_tokens, output_kind)`` returns raw
32    text. Only input images and the public prompt reach the callable; answers,
33    evidence and target boxes remain inside the evaluator. Two-image requests
34    preserve reference-then-target order and use the verified reference crop.
35
36    The generic bbox contract is one JSON object with ``label`` and ``bbox``
37    (XYXY, normalized 0–1000). Binary questions require ``yes`` or ``no``.
38    Malformed outputs and inference exceptions score as failures and remain
39    in the denominator. Dataset download errors abort rather than dropping rows.
40    Use a fresh output directory for each run; no silent overwrites or resume.
41    """
42    if not model_name.strip() or not model_revision.strip():
43        raise ConfigError("model_name and model_revision must identify the evaluated model")
44    manifest_path = Path(manifest_path)
45    samples = load_manifest(manifest_path)
46    if not samples:
47        raise ManifestError("VQA manifest contains no samples")
48    output = Path(output_dir)
49    if output.exists() and any(output.iterdir()):
50        raise ConfigError(f"Use an empty output directory: {output}")
51    output.mkdir(parents=True, exist_ok=True)
52    config = {
53        "model": model_name,
54        "model_revision": model_revision,
55        "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
56        "num_samples": len(samples),
57        "protocol": "generic_single_call",
58        "timing": "end_to_end_callable_wall_ms_including_API_transport",
59    }
60    (output / "run_config.json").write_text(json.dumps(config, indent=2) + "\n")
61    scored = []
62    with (output / "predictions.jsonl").open("w", encoding="utf-8") as handle:
63        for sample in samples:
64            images = fetch_images(sample, image_cache)
65            prompt = build_prompt(sample, "custom")
66            started = time.perf_counter()
67            raw = None
68            error = None
69            try:
70                raw = predict(images, prompt.text, prompt.max_new_tokens, prompt.output_kind)
71                if not isinstance(raw, str):
72                    raise AdapterError("The VQA callable must return raw text")
73                parsed = parse_output(sample, raw, "custom")
74            except Exception as exc:
75                error = f"{type(exc).__name__}: {exc}"
76                parsed = ParsedOutput(False, error=error)
77            elapsed_ms = (time.perf_counter() - started) * 1000
78            scored.append((sample, parsed))
79            row = {
80                "sample_id": sample.sample_id,
81                "raw_output": raw if isinstance(raw, str) else None,
82                "parsed": asdict(parsed),
83                "inference_error": error,
84                "callable_wall_ms": elapsed_ms,
85            }
86            handle.write(json.dumps(row, allow_nan=False) + "\n")
87            handle.flush()
88    report = {**config, "metrics": score_predictions(scored)}
89    (output / "result.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
90    return report

240def build_prompt(sample: VQASample, model_key: str) -> PromptSpec:
241    question = display_question(sample.question)
242    if sample.question_type in BINARY_TYPES:
243        if model_key.startswith("paligemma2-"):
244            return PromptSpec(f"answer en {question}\n", 8, "binary")
245        return PromptSpec(
246            f"{question}\nAnswer using exactly one lowercase word: yes or no.", 4, "binary"
247        )
248    if sample.question_type in BBOX_TYPES:
249        if model_key.startswith("cosmos-reason2-"):
250            return PromptSpec(
251                _cosmos_reason2_bbox_instruction(question),
252                128,
253                "bbox_json_normalized_1000",
254            )
255        if model_key.startswith("deepseek-vl2"):
256            expression = native_referring_expression(sample)
257            if sample.is_in_context:
258                # DeepSeek-VL2 documents <|ref|> for ordinary single-image
259                # referring-expression localization, but its native
260                # multi-image/in-context grounding examples use the
261                # <|grounding|> mode.  Image 1 is already the hash-verified
262                # reference crop; Image 2 is the target scene.
263                return PromptSpec(
264                    "<|grounding|>The first image is a crop showing the reference "
265                    f"object. Locate only {expression} and return its grounding in "
266                    "the second image.",
267                    128,
268                    "bbox_native_question_grounding",
269                )
270            return PromptSpec(
271                f"<|ref|>{expression}<|/ref|>.",
272                128,
273                "bbox_native_question_grounding",
274            )
275        if model_key.startswith("florence2-"):
276            # One scored native call: a deterministic, GT-free rewrite of the
277            # question is supplied to caption-to-phrase grounding. Only the
278            # returned bbox is evaluated; no answer-label call is involved.
279            return PromptSpec(
280                native_referring_expression(sample),
281                128,
282                "bbox_native_question_grounding",
283            )
284        if model_key.startswith("paligemma2-"):
285            # PaliGemma's detector accepts a referring phrase rather than an
286            # interrogative. Rewrite question syntax without using its answer,
287            # then perform selection/localization in one generation.
288            expression = native_referring_expression(sample)
289            return PromptSpec(f"detect {expression}\n", 64, "bbox_native_question_grounding")
290        if model_key in {"molmo-7b-d", "molmoe-1b"}:
291            return PromptSpec(
292                _molmo_bbox_instruction(question),
293                96,
294                "bbox_molmo_percent_100",
295            )
296        if model_key in {"internvl3.5-1b", "internvl3.5-14b"}:
297            return PromptSpec(
298                _internvl_bbox_instruction(native_referring_expression(sample)),
299                96,
300                "bbox_native_internvl_grounding",
301            )
302        return PromptSpec(
303            _bbox_instruction(question),
304            96,
305            "bbox_json_normalized_1000",
306        )
307    raise ConfigError(
308        f"unsupported VQA question type: {sample.question_type}",
309        hint="use a type in the frozen VQA task contract",
310    )

325def load_manifest(path: str | Path) -> list[VQASample]:
326    records: list[VQASample] = []
327    with Path(path).open(encoding="utf-8") as handle:
328        for line_number, line in enumerate(handle, 1):
329            if not line.strip():
330                continue
331            try:
332                records.append(VQASample.from_dict(json.loads(line)))
333            except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
334                raise ManifestError(
335                    f"invalid VQA manifest row at {path}:{line_number}: {exc}",
336                    hint="regenerate the manifest from the source parquets",
337                ) from exc
338    if not records:
339        raise ManifestError(f"empty VQA manifest: {path}", hint="provide at least one sample")
340    ids = [record.sample_id for record in records]
341    if len(ids) != len(set(ids)):
342        raise ManifestError(
343            "duplicate sample_id in VQA manifest", hint="deduplicate source questions"
344        )
345    return records

151def parse_output(
152    sample: VQASample,
153    raw: str,
154    model_key: str,
155    *,
156    paligemma_target_only: bool = False,
157) -> ParsedOutput:
158    label: str | None
159    match: re.Match[str] | None
160    if sample.question_type in BINARY_TYPES:
161        token = normalize_label(raw)
162        if token not in {"yes", "no"}:
163            return ParsedOutput(False, error="expected exactly yes or no")
164        return ParsedOutput(True, label=token)
165    if sample.question_type in ATTRIBUTE_TYPES:
166        label = normalize_label(raw)
167        if not label or len(label.split()) > 8:
168            return ParsedOutput(False, error="invalid object-name answer")
169        return ParsedOutput(True, label=label)
170    if sample.question_type in BBOX_TYPES and model_key.startswith("cosmos-reason2-"):
171        candidates = _cosmos_bbox_candidates(raw)
172        if len(candidates) != 1:
173            return ParsedOutput(
174                False,
175                error=f"expected exactly one Cosmos bbox_2d candidate, got {len(candidates)}",
176            )
177        value = candidates[0]
178        try:
179            raw_bbox = value["bbox_2d"]
180            if not isinstance(raw_bbox, list) or len(raw_bbox) != 4:
181                raise AdapterError(
182                    "Cosmos bbox_2d must contain four numbers",
183                    hint="return one normalized XYXY box",
184                )
185            bbox = tuple(float(coordinate) for coordinate in raw_bbox)
186            if not all(float("-inf") < coordinate < float("inf") for coordinate in bbox):
187                raise AdapterError("Cosmos bbox contains a non-finite number")
188            if not all(0 <= coordinate <= 1000 for coordinate in bbox):
189                raise AdapterError("Cosmos bbox is outside the 0-1000 grid")
190            x0, y0, x1, y1 = bbox
191            if not (x0 <= x1 and y0 <= y1):
192                raise AdapterError("Cosmos bbox corners are not ordered XYXY")
193            scaled = (
194                x0 * (sample.img_w - 1) / 1000,
195                y0 * (sample.img_h - 1) / 1000,
196                x1 * (sample.img_w - 1) / 1000,
197                y1 * (sample.img_h - 1) / 1000,
198            )
199            label = normalize_label(str(value.get("label", ""))) or None
200            return ParsedOutput(
201                True,
202                label=label,
203                bbox=scaled,
204                coordinate_format="cosmos_bbox_2d_0_1000",
205            )
206        except (KeyError, TypeError, ValueError, RPXError) as exc:
207            return ParsedOutput(False, error=str(exc))
208    if sample.question_type in BBOX_TYPES and model_key.startswith("internvl3.5-"):
209        # InternVL's official RefCOCO evaluation protocol serializes one
210        # target-relative 0--1000 box.  Its own evaluator deliberately accepts
211        # either ``label[[x0,y0,x1,y1]]`` or ``[x0,y0,x1,y1]``.  Accept those
212        # two serializations before the generic JSON fallback, but never choose
213        # silently among multiple candidates in this one-object benchmark.
214        matches = list(_INTERNVL_BBOX_RE.finditer(raw))
215        if matches:
216            if len(matches) != 1:
217                return ParsedOutput(False, error="expected exactly one InternVL bbox")
218            match = matches[0]
219            bbox = tuple(float(value) for value in match.groups())
220            if not all(0 <= value <= 1000 for value in bbox):
221                return ParsedOutput(False, error="InternVL bbox is outside the 0-1000 grid")
222            x0, y0, x1, y1 = bbox
223            if not (x0 <= x1 and y0 <= y1):
224                return ParsedOutput(False, error="InternVL bbox corners are not ordered XYXY")
225            prefix = raw[: match.start()]
226            prefix = re.sub(r"</?(?:ref|box)>", "", prefix, flags=re.IGNORECASE)
227            label = normalize_label(prefix.splitlines()[-1]) or None
228            return ParsedOutput(
229                True,
230                label=label,
231                bbox=(
232                    x0 * (sample.img_w - 1) / 1000,
233                    y0 * (sample.img_h - 1) / 1000,
234                    x1 * (sample.img_w - 1) / 1000,
235                    y1 * (sample.img_h - 1) / 1000,
236                ),
237                coordinate_format="internvl_bbox_0_1000",
238            )
239        # The 1B checkpoint sometimes preserves the same 0--1000 coordinate
240        # contract but spells the single box out as labelled corners.  This is
241        # a deterministic serialization variant, not box repair: require one
242        # complete pair, validate it identically, and never infer coordinates
243        # from descriptive prose.
244        verbose_matches = re.findall(
245            r"top[\s-]*left\s+corner\s*:\s*[\[(]\s*"
246            r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*[\])]"
247            r".*?bottom[\s-]*right\s+corner\s*:\s*[\[(]\s*"
248            r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*[\])]",
249            raw,
250            flags=re.IGNORECASE | re.DOTALL,
251        )
252        if verbose_matches:
253            if len(verbose_matches) != 1:
254                return ParsedOutput(False, error="expected exactly one InternVL corner pair")
255            bbox = tuple(float(value) for value in verbose_matches[0])
256            if not all(0 <= value <= 1000 for value in bbox):
257                return ParsedOutput(False, error="InternVL bbox is outside the 0-1000 grid")
258            x0, y0, x1, y1 = bbox
259            if not (x0 <= x1 and y0 <= y1):
260                return ParsedOutput(False, error="InternVL bbox corners are not ordered XYXY")
261            return ParsedOutput(
262                True,
263                bbox=(
264                    x0 * (sample.img_w - 1) / 1000,
265                    y0 * (sample.img_h - 1) / 1000,
266                    x1 * (sample.img_w - 1) / 1000,
267                    y1 * (sample.img_h - 1) / 1000,
268                ),
269                coordinate_format="internvl_verbose_bbox_0_1000",
270            )
271        # Retain the strict JSON fallback for a checkpoint that follows the
272        # explicit coordinate contract but emits the generic RPX envelope.
273    if sample.question_type in BBOX_TYPES and model_key.startswith("paligemma2-"):
274        match = _LOC_RE.search(raw)
275        if match:
276            if sample.is_in_context and not paligemma_target_only:
277                return ParsedOutput(
278                    False,
279                    error=(
280                        "native PaliGemma loc tokens are relative to the labelled "
281                        "two-image composite, not Image 2; return target-relative JSON"
282                    ),
283                )
284            vals = {key: int(value) for key, value in match.groupdict().items()}
285            bbox = (
286                vals["x0"] * sample.img_w / 1024,
287                vals["y0"] * sample.img_h / 1024,
288                vals["x1"] * sample.img_w / 1024,
289                vals["y1"] * sample.img_h / 1024,
290            )
291            label = normalize_label(raw[match.end() :].split("<", 1)[0]) or None
292            return ParsedOutput(
293                True, label=label, bbox=bbox, coordinate_format="paligemma_loc_1024"
294            )
295        # The scored single-call prompt requests the same JSON contract used
296        # by the other VLMs. Fall through to that parser when PaliGemma obeys
297        # it; native <loc> remains accepted as an architecture-specific bbox
298        # serialization, but no second model call is made.
299    if sample.question_type in BBOX_TYPES:
300        try:
301            value = _json_object(raw)
302            raw_bbox = value["bbox"]
303            # Some otherwise-correct VLM responses add one redundant list
304            # dimension.  Unwrap exactly one singleton; do not guess any more
305            # complicated structure.
306            if (
307                isinstance(raw_bbox, list)
308                and len(raw_bbox) == 1
309                and isinstance(raw_bbox[0], list)
310            ):
311                raw_bbox = raw_bbox[0]
312            if not isinstance(raw_bbox, list) or len(raw_bbox) != 4:
313                raise AdapterError(
314                    "bbox must contain four numbers", hint="return normalized xyxy coordinates"
315                )
316            bbox = tuple(float(v) for v in raw_bbox)
317            if not all(float("-inf") < value < float("inf") for value in bbox):
318                raise AdapterError("bbox contains a non-finite number", hint="return finite pixels")
319            # Molmo's model-facing bbox protocol uses the same 0--100
320            # percentage grid as its documented spatial pointing interface.
321            # This is a complete four-corner box, not a point-derived box.
322            # Convert it deterministically here while retaining the untouched
323            # model text in raw_output for auditability.
324            molmo_percent = model_key in {"molmo-7b-d", "molmoe-1b"}
325            if molmo_percent and not all(0 <= coordinate <= 100 for coordinate in bbox):
326                raise AdapterError(
327                    "Molmo bbox is outside percentage 0-100 coordinates",
328                    hint="return percentage xyxy coordinates",
329                )
330
331            # JSON produced by an adapter may contain a machine-rounding
332            # residue at an exact normalized endpoint (for example
333            # 999 * 1000 / 999 -> 1000.0000000000001). Accept only that tiny
334            # numerical residue, then clamp it; genuinely out-of-range model
335            # coordinates remain invalid.
336            coordinate_epsilon = 1e-9
337            if not all(
338                -coordinate_epsilon <= coordinate <= 1000 + coordinate_epsilon
339                for coordinate in bbox
340            ):
341                raise AdapterError(
342                    "bbox is outside normalized 0-1000 coordinates",
343                    hint="return normalized xyxy coordinates",
344                )
345            bbox = tuple(min(1000.0, max(0.0, coordinate)) for coordinate in bbox)
346            # Decode the two widespread normalized conventions without using
347            # model identity. Values containing a genuine fraction and entirely
348            # within [0,1] are unit-normalized; everything else follows the
349            # requested [0,1000] protocol. This fixes the previous silent bug
350            # where [0.5,...] became a sub-pixel box at the top-left.
351            unit_normalized = not molmo_percent and all(
352                0 <= coordinate <= 1 for coordinate in bbox
353            )
354            denominator = 100 if molmo_percent else 1 if unit_normalized else 1000
355            coordinate_format = (
356                "molmo_bbox_percent_0_100"
357                if molmo_percent
358                else "normalized_0_1"
359                if unit_normalized
360                else "normalized_0_1000"
361            )
362            bbox = (
363                bbox[0] * (sample.img_w - 1) / denominator,
364                bbox[1] * (sample.img_h - 1) / denominator,
365                bbox[2] * (sample.img_w - 1) / denominator,
366                bbox[3] * (sample.img_h - 1) / denominator,
367            )
368            x0, y0, x1, y1 = bbox
369            if not (0 <= x0 <= x1 < sample.img_w and 0 <= y0 <= y1 < sample.img_h):
370                raise AdapterError(
371                    "bbox is outside the original image", hint="undo processor resizing first"
372                )
373            label = normalize_label(str(value.get("label", ""))) or None
374            return ParsedOutput(
375                True, label=label, bbox=bbox, coordinate_format=coordinate_format
376            )
377        except (KeyError, TypeError, ValueError, json.JSONDecodeError, RPXError) as exc:
378            return ParsedOutput(False, error=str(exc))
379    return ParsedOutput(False, error="unsupported task")

111def score_predictions(
112    pairs: Iterable[tuple[VQASample, ParsedOutput]], iou_threshold: float = 0.5
113) -> dict[str, Any]:
114    rows = list(pairs)
115    if not rows:
116        raise MetricError("no VQA predictions to score", hint="provide one output per request")
117    valid = [float(parsed.valid) for _, parsed in rows]
118    by_type: dict[str, list[float]] = defaultdict(list)
119    binary: list[float] = []
120    attribute: list[float] = []
121    ious: list[float] = []
122    bbox_hits: list[float] = []
123    bbox_labels: list[float] = []
124    bbox_label_token_f1: list[float] = []
125    bbox_joint_hits: list[float] = []
126    bbox_validity: list[float] = []
127    bbox_gious: list[float] = []
128    bbox_valid_gious: list[float] = []
129    bbox_center_hits: list[float] = []
130    bbox_hits_by_threshold: dict[float, list[float]] = {
131        threshold: [] for threshold in (0.25, 0.5, 0.75, *COCO_IOU_THRESHOLDS)
132    }
133    detailed_by_type: dict[str, list[dict[str, float]]] = defaultdict(list)
134    for sample, parsed in rows:
135        score = 0.0
136        if parsed.valid and sample.question_type in BINARY_TYPES:
137            score = float(parsed.label == sample.answer)
138            binary.append(score)
139        elif parsed.valid and sample.question_type in ATTRIBUTE_TYPES:
140            score = float(parsed.label == normalize_label(sample.answer))
141            attribute.append(score)
142        elif sample.question_type in BBOX_TYPES:
143            assert sample.answer_bbox is not None
144            bbox_valid = bool(
145                parsed.valid
146                and bbox_is_valid(parsed.bbox, sample.img_w, sample.img_h)
147            )
148            iou = 0.0
149            giou = -1.0
150            center_hit = 0.0
151            if bbox_valid and parsed.bbox is not None:
152                iou = bbox_iou(parsed.bbox, sample.answer_bbox)
153                giou = bbox_generalized_iou(parsed.bbox, sample.answer_bbox)
154                center_hit = float(
155                    bbox_center_in_ground_truth(parsed.bbox, sample.answer_bbox)
156                )
157            ious.append(iou)
158            bbox_validity.append(float(bbox_valid))
159            bbox_gious.append(giou)
160            if bbox_valid:
161                bbox_valid_gious.append(giou)
162            bbox_center_hits.append(center_hit)
163            score = float(iou >= iou_threshold)
164            bbox_hits.append(score)
165            label_exact = float(parsed.label == normalize_label(sample.answer))
166            label_f1 = label_token_f1(parsed.label, sample.answer)
167            # Missing/unparseable labels count as zero; excluding them would
168            # inflate textual correctness precisely when protocol compliance
169            # is poor.
170            bbox_labels.append(label_exact)
171            bbox_label_token_f1.append(label_f1)
172            bbox_joint_hits.append(float(label_exact == 1.0 and iou >= iou_threshold))
173            for threshold, values in bbox_hits_by_threshold.items():
174                values.append(float(iou >= threshold))
175            detailed_by_type[sample.question_type].append(
176                {
177                    "valid": float(parsed.valid),
178                    "bbox_valid": float(bbox_valid),
179                    "iou": iou,
180                    "giou": giou,
181                    "center_in_gt": center_hit,
182                    "label_exact": label_exact,
183                    "label_token_f1": label_f1,
184                }
185            )
186        by_type[sample.question_type].append(score)
187    per_type_detailed = {}
188    for question_type, detail_rows in sorted(detailed_by_type.items()):
189        per_type_detailed[question_type] = {
190            "count": len(detail_rows),
191            "parse_rate": _mean([value["valid"] for value in detail_rows]),
192            "bbox_validity_rate": _mean([value["bbox_valid"] for value in detail_rows]),
193            "bbox_mean_iou": _mean([value["iou"] for value in detail_rows]),
194            "bbox_mean_giou": _mean([value["giou"] for value in detail_rows]),
195            "bbox_mean_giou_valid": _mean(
196                [value["giou"] for value in detail_rows if value["bbox_valid"]]
197            ),
198            "bbox_center_in_gt_accuracy": _mean(
199                [value["center_in_gt"] for value in detail_rows]
200            ),
201            "bbox_accuracy_at_0_25": _mean(
202                [float(value["iou"] >= 0.25) for value in detail_rows]
203            ),
204            "bbox_accuracy_at_0_5": _mean(
205                [float(value["iou"] >= 0.5) for value in detail_rows]
206            ),
207            "bbox_accuracy_at_0_75": _mean(
208                [float(value["iou"] >= 0.75) for value in detail_rows]
209            ),
210            "label_normalized_exact_match": _mean(
211                [value["label_exact"] for value in detail_rows]
212            ),
213            "label_token_f1": _mean([value["label_token_f1"] for value in detail_rows]),
214        }
215    return {
216        "count": len(rows),
217        "primary_metric": "bbox_accuracy_at_0_5",
218        "bbox_scoring_uses_generated_label": False,
219        "parse_rate": _mean(valid),
220        "binary_accuracy": _mean(binary),
221        "attribute_exact_match": _mean(attribute),
222        "bbox_count": len(ious),
223        "bbox_validity_rate": _mean(bbox_validity),
224        "bbox_accuracy_at_0_5": _mean(bbox_hits),
225        "bbox_accuracy_at_0_25": _mean(bbox_hits_by_threshold[0.25]),
226        "bbox_accuracy_at_0_75": _mean(bbox_hits_by_threshold[0.75]),
227        "bbox_mean_iou": _mean(ious),
228        "bbox_mean_giou": _mean(bbox_gious),
229        "bbox_mean_giou_valid": _mean(bbox_valid_gious),
230        "bbox_center_in_gt_accuracy": _mean(bbox_center_hits),
231        "bbox_mean_accuracy_50_95": _mean(
232            [_mean(bbox_hits_by_threshold[threshold]) for threshold in COCO_IOU_THRESHOLDS]
233        ),
234        # Retain the old key for report compatibility, but make its exact
235        # single-reference meaning explicit in the new key.
236        "bbox_label_accuracy": _mean(bbox_labels),
237        "bbox_label_normalized_exact_match": _mean(bbox_labels),
238        "bbox_label_token_f1": _mean(bbox_label_token_f1),
239        "bbox_joint_label_exact_and_iou_at_0_5": _mean(bbox_joint_hits),
240        "per_type": {key: _mean(value) for key, value in sorted(by_type.items())},
241        "per_type_detailed": per_type_detailed,
242        "metric_notes": {
243            "bbox_validity_rate": (
244                "Fraction of bbox-task rows with a parsed, finite, ordered inclusive-XYXY "
245                "box wholly inside the target image."
246            ),
247            "bbox_mean_giou": (
248                "Mean generalized IoU over every bbox-task row; an invalid or missing box "
249                "receives -1.0, the worst possible GIoU, to prevent parse failures from "
250                "inflating localization quality."
251            ),
252            "bbox_mean_giou_valid": (
253                "Diagnostic mean GIoU over valid boxes only; use bbox_mean_giou for the "
254                "all-row model comparison."
255            ),
256            "bbox_center_in_gt_accuracy": (
257                "Fraction of every bbox-task row whose predicted-box center lies inside "
258                "the inclusive ground-truth box; invalid or missing boxes count as misses."
259            ),
260            "bbox_mean_accuracy_50_95": (
261                "Mean single-box success rate over IoU thresholds 0.50:0.05:0.95; "
262                "not detection mAP because predictions have no confidence scores."
263            ),
264            "bbox_label_normalized_exact_match": (
265                "Informational only: one-reference normalized exact match; bbox scoring "
266                "does not depend on the generated label."
267            ),
268            "bbox_label_token_f1": (
269                "Informational only: lexical token overlap; bbox scoring does not depend "
270                "on the generated label."
271            ),
272            "bbox_joint_label_exact_and_iou_at_0_5": (
273                "Informational diagnostic only; the benchmark's bbox result is "
274                "bbox_accuracy_at_0_5."
275            ),
276        },
277    }