rpx_benchmark.vqa.prompts

  1"""Task prompts before each model's official chat/image template is applied."""
  2
  3from __future__ import annotations
  4
  5from dataclasses import dataclass
  6
  7from ..exceptions import ConfigError
  8from .contract import BBOX_TYPES, BINARY_TYPES, VQASample
  9
 10
 11@dataclass(frozen=True)
 12class PromptSpec:
 13    text: str
 14    max_new_tokens: int
 15    output_kind: str
 16
 17
 18def display_question(question: str) -> str:
 19    """Hide catalog encoding without changing the stored ground truth label."""
 20    return " ".join(question.replace("_", " ").split())
 21
 22
 23def native_referring_expression(sample: VQASample) -> str:
 24    """Rewrite an RPX question into a native grounding expression.
 25
 26    Florence/PaliGemma grounding heads are trained on referring expressions,
 27    not interrogative VQA syntax.  This transformation is deterministic and
 28    uses only the public question/type -- never the answer or GT bbox -- so the
 29    scored path remains one model generation.
 30    """
 31    question = display_question(sample.question).strip().rstrip("?")
 32    lower = question.lower()
 33    if sample.question_type == "spatial_lr_extreme":
 34        if "left" in lower:
 35            return "the leftmost object"
 36        if "right" in lower:
 37            return "the rightmost object"
 38    if sample.question_type == "depth_closest" and "closest" in lower and "camera" in lower:
 39        return "the object closest to the camera"
 40    if sample.question_type == "spatial_farthest":
 41        marker = "farthest from the "
 42        if marker in lower:
 43            return "the object farthest from " + question[lower.index(marker) + len(marker) :]
 44    if sample.question_type == "inctx_spatial_farthest":
 45        return "the object in image 2 farthest from the object shown in image 1"
 46
 47    # Normal attribute questions all begin with one of these stable forms.
 48    for prefix in ("which object is ", "what is the object "):
 49        if lower.startswith(prefix):
 50            description = question[len(prefix) :]
 51            if sample.question_type == "attr_single_color":
 52                return "the " + description + " object"
 53            return "the object " + description
 54
 55    # In-context questions already encode the complete reference relation;
 56    # remove only the redundant request for output formatting and turn the
 57    # interrogative into a grammatical relative clause.
 58    suffix = "? What is its bounding box in Image 2"
 59    if question.endswith(suffix):
 60        question = question[: -len(suffix)]
 61    prefix = "Which object in Image 2 "
 62    if question.startswith(prefix):
 63        relation = question[len(prefix) :]
 64        if relation.startswith("has "):
 65            relation = "that " + relation
 66        elif relation.startswith("is "):
 67            relation = "that " + relation
 68        return "the object in image 2 " + relation
 69    if question.lower().startswith("which object "):
 70        return "the object " + question[len("Which object ") :]
 71    return question
 72
 73
 74def _bbox_instruction(question: str) -> str:
 75    """Direct, model-neutral bbox instruction for every JSON-capable VLM.
 76
 77    The scored path is deliberately one model call: the model sees the original
 78    question and every required image while choosing and localizing the answer.
 79    Splitting this into answer-then-ground calls compounds two independent model
 80    errors and no longer measures the dataset's end-to-end task.
 81    """
 82    return (
 83        f"{question}\n"
 84        "Identify and localize the one visible object that answers the question. "
 85        "For a relational question, localize the result object, not the reference "
 86        "object named or shown in the question. The answer object is in the target "
 87        "image (Image 2 for a two-image input; otherwise the only image). Return only "
 88        'JSON: {"label":"object name","bbox":[x_min,y_min,x_max,y_max]}. '
 89        "Normalize every bbox coordinate from 0 to 1000 relative to the target "
 90        "image width and height. Use XYXY corner order and enclose the entire object. "
 91        "No Markdown or explanation."
 92    )
 93
 94
 95def _molmo_bbox_instruction(question: str) -> str:
 96    """Request an entire box on Molmo's native percentage coordinate grid.
 97
 98    Molmo's documented spatial markup uses image-relative percentages.  Asking
 99    these checkpoints for the generic RPX 0--1000 grid made otherwise plausible
100    percentage boxes score as tiny top-left boxes.  Keep the scored task to one
101    model call, but make the model-facing coordinate convention explicit; the
102    output parser performs the deterministic 0--100 to image-pixel conversion.
103    """
104    return (
105        f"{question}\n"
106        "Identify and localize the one visible object that answers the question. "
107        "For a relational question, localize the result object, not the reference "
108        "object named or shown in the question. The answer object is in the target "
109        "image (Image 2 for a two-image input; otherwise the only image). Return only "
110        '{"label":"object name","bbox":[x_min,y_min,x_max,y_max]}. '
111        "Express every bbox coordinate as a percentage from 0 to 100 relative to "
112        "the target image width and height. Use XYXY corner order and enclose the "
113        "entire object. No Markdown or explanation."
114    )
115
116
117def _internvl_bbox_instruction(expression: str) -> str:
118    """Use InternVL 3.5's trained RefCOCO-style native grounding protocol."""
119    # Match OpenGVLab's RefCOCO evaluation prompt exactly.  Output constraints
120    # added after this sentence can knock a native grounding checkpoint out of
121    # its trained response protocol; strictness belongs in the parser instead.
122    return (
123        "Please provide the bounding box coordinate of the region this sentence "
124        f"describes: <ref>{expression}</ref>"
125    )
126
127
128def _cosmos_reason2_bbox_instruction(question: str) -> str:
129    """Use a strict Reason2 2D-grounding JSON and coordinate grid.
130
131    NVIDIA's official Reason2 grounding prompt asks to locate a bounding box
132    and return JSON. RPX makes that underspecified JSON contract explicit using
133    Qwen3-VL's ``bbox_2d`` field and target-relative 0--1000 XYXY coordinates;
134    Reason2 is a Qwen3-VL architecture. RPX also adds the GT-free selection
135    rules required by relational questions. The optional Reason2 chain-of-thought
136    instruction is deliberately omitted: this benchmark scores one direct answer
137    and retains deterministic decoding.
138    """
139    return (
140        f"{question}\n"
141        "Identify the one visible object that answers the question. For a relational "
142        "question, select the result object, not the reference object named or shown "
143        "in the question. The answer object is in the target image (Image 2 for a "
144        "two-image input; otherwise the only image). Locate the accurate bounding box "
145        "of that answer object. Return only a JSON list in this exact form: "
146        '[{"bbox_2d":[x1,y1,x2,y2],"label":"object name"}]. '
147        "Normalize x and y independently from 0 to 1000 relative to the target image. "
148        "Use XYXY corner order and enclose the entire object. No Markdown, reasoning, "
149        "or explanation."
150    )
151
152
153def build_semantic_diagnostic_prompt(sample: VQASample, model_key: str) -> PromptSpec:
154    """Ask for the answer label only, without leaking the ground truth.
155
156    This is deliberately diagnostic rather than a scored benchmark protocol:
157    it separates target selection/reasoning from coordinate prediction.
158    """
159    question = display_question(sample.question)
160    if model_key.startswith("florence2-"):
161        # Florence task tokens are a closed native interface.  Extra formatting
162        # prose is treated as VQA question text and can be echoed by the model.
163        return PromptSpec(question, 32, "diagnostic_semantic_label")
164    if model_key.startswith("paligemma2-"):
165        # Follow the checkpoint's documented `answer {lang} {question}` form
166        # exactly.  Enforce short-answer formatting in the diagnostic parser.
167        return PromptSpec(f"answer en {question}\n", 32, "diagnostic_semantic_label")
168    instruction = (
169        f"{question}\nIdentify the one object that answers the question. "
170        "For a relational question, name the result object, not the reference "
171        "object named or shown in the question. Return only the shortest common "
172        "noun phrase naming that object: 1 to 5 words, with no sentence, article, "
173        "coordinates, Markdown, punctuation, or explanation."
174    )
175    return PromptSpec(instruction, 32, "diagnostic_semantic_label")
176
177
178def build_oracle_localization_prompt(sample: VQASample, model_key: str) -> PromptSpec:
179    """Localize the known GT label in the target image for diagnosis only."""
180    return build_label_localization_prompt(display_question(sample.answer), model_key, oracle=True)
181
182
183def build_label_localization_prompt(
184    label: str, model_key: str, *, oracle: bool = False
185) -> PromptSpec:
186    """Ground a label/referring phrase without canonicalizing its wording.
187
188    The predicted-label diagnostic deliberately passes the model's own answer
189    phrase through unchanged.  Correctness is determined spatially against the
190    target identity, so synonyms and harmless modifiers cannot create a false
191    semantic failure merely because their strings differ from the catalog.
192    """
193    label = display_question(label)
194    output_kind = "diagnostic_oracle_bbox" if oracle else "diagnostic_predicted_label_bbox"
195    if model_key.startswith("florence2-"):
196        # Florence's official phrase-grounding interface is TASK + phrase.
197        # Adding JSON instructions makes every noun in those instructions a
198        # grounding candidate and was the cause of the invalid smoke results.
199        return PromptSpec(label, 128, output_kind)
200    if model_key.startswith("paligemma2-"):
201        return PromptSpec(f"detect {label}\n", 64, output_kind)
202    if model_key.startswith("deepseek-vl2"):
203        return PromptSpec(f"<|ref|>{label}<|/ref|>.", 128, output_kind)
204    if model_key.startswith("cosmos-reason2-"):
205        return PromptSpec(
206            f'Locate the accurate bounding box of "{label}" in the target image. '
207            "Return only a JSON list in this exact form: "
208            '[{"bbox_2d":[x1,y1,x2,y2],"label":"object name"}]. '
209            "Normalize x and y independently from 0 to 1000 relative to the target "
210            "image. Use XYXY corner order. No Markdown, reasoning, or explanation.",
211            96,
212            output_kind,
213        )
214    if model_key in {"molmo-7b-d", "molmoe-1b"}:
215        # Molmo 0924 is natively trained to answer pointing prompts with
216        # <point>/<points> markup on a 0--100 coordinate grid.  This remains
217        # an explicitly unscored localization diagnostic: a point is never
218        # fabricated into a bbox or substituted for the scored RPX result.
219        return PromptSpec(
220            f"Point to the {label} in the image.",
221            64,
222            (
223                "diagnostic_oracle_point"
224                if oracle
225                else "diagnostic_predicted_label_point"
226            ),
227        )
228    return PromptSpec(
229        f'Locate the visible object named "{label}" in the target image. '
230        'Return only JSON: {"label":"object name","bbox":'
231        "[x_min,y_min,x_max,y_max]}. Normalize every bbox coordinate from 0 "
232        "to 1000 relative to the target-image width and height. Use XYXY corner "
233        "order and enclose the entire object. No Markdown or explanation.",
234        96,
235        output_kind,
236    )
237
238
239def build_prompt(sample: VQASample, model_key: str) -> PromptSpec:
240    question = display_question(sample.question)
241    if sample.question_type in BINARY_TYPES:
242        if model_key.startswith("paligemma2-"):
243            return PromptSpec(f"answer en {question}\n", 8, "binary")
244        return PromptSpec(
245            f"{question}\nAnswer using exactly one lowercase word: yes or no.", 4, "binary"
246        )
247    if sample.question_type in BBOX_TYPES:
248        if model_key.startswith("cosmos-reason2-"):
249            return PromptSpec(
250                _cosmos_reason2_bbox_instruction(question),
251                128,
252                "bbox_json_normalized_1000",
253            )
254        if model_key.startswith("deepseek-vl2"):
255            expression = native_referring_expression(sample)
256            if sample.is_in_context:
257                # DeepSeek-VL2 documents <|ref|> for ordinary single-image
258                # referring-expression localization, but its native
259                # multi-image/in-context grounding examples use the
260                # <|grounding|> mode.  Image 1 is already the hash-verified
261                # reference crop; Image 2 is the target scene.
262                return PromptSpec(
263                    "<|grounding|>The first image is a crop showing the reference "
264                    f"object. Locate only {expression} and return its grounding in "
265                    "the second image.",
266                    128,
267                    "bbox_native_question_grounding",
268                )
269            return PromptSpec(
270                f"<|ref|>{expression}<|/ref|>.",
271                128,
272                "bbox_native_question_grounding",
273            )
274        if model_key.startswith("florence2-"):
275            # One scored native call: a deterministic, GT-free rewrite of the
276            # question is supplied to caption-to-phrase grounding. Only the
277            # returned bbox is evaluated; no answer-label call is involved.
278            return PromptSpec(
279                native_referring_expression(sample),
280                128,
281                "bbox_native_question_grounding",
282            )
283        if model_key.startswith("paligemma2-"):
284            # PaliGemma's detector accepts a referring phrase rather than an
285            # interrogative. Rewrite question syntax without using its answer,
286            # then perform selection/localization in one generation.
287            expression = native_referring_expression(sample)
288            return PromptSpec(f"detect {expression}\n", 64, "bbox_native_question_grounding")
289        if model_key in {"molmo-7b-d", "molmoe-1b"}:
290            return PromptSpec(
291                _molmo_bbox_instruction(question),
292                96,
293                "bbox_molmo_percent_100",
294            )
295        if model_key in {"internvl3.5-1b", "internvl3.5-14b"}:
296            return PromptSpec(
297                _internvl_bbox_instruction(native_referring_expression(sample)),
298                96,
299                "bbox_native_internvl_grounding",
300            )
301        return PromptSpec(
302            _bbox_instruction(question),
303            96,
304            "bbox_json_normalized_1000",
305        )
306    raise ConfigError(
307        f"unsupported VQA question type: {sample.question_type}",
308        hint="use a type in the frozen VQA task contract",
309    )