rpx_benchmark.runner

  1"""Benchmark runner orchestrating model, data, and metrics."""
  2
  3from __future__ import annotations
  4
  5import time
  6from collections import defaultdict
  7from typing import Any, Callable, Dict, List, Optional, Tuple
  8
  9import numpy as np
 10
 11from .api import (
 12    BenchmarkModel,
 13    DepthPrediction,
 14    Difficulty,
 15    Phase,
 16    TaskType,
 17    validate_prediction,
 18)
 19from .deployment import (
 20    DeploymentReadinessReport,
 21    StackGeometricCoherenceResult,
 22    TemporalStabilityResult,
 23    compute_str,
 24    compute_weighted_phase_score,
 25)
 26from .evaluators import BenchmarkResult, MetricSuite
 27from .exceptions import ModelError
 28from .loader import RPXDataset
 29from .logging_utils import get_logger
 30from .metrics.depth_alignment import align_pred_to_gt_pooled
 31from .profiler import EfficiencyMetadata, LatencyProfiler, MemoryProfiler
 32
 33log = get_logger(__name__)
 34
 35ProgressCallback = Callable[[int, Optional[int], str], None]
 36"""Signature: ``progress(done, total, stage) -> None``.
 37
 38``total`` may be ``None`` while the total is unknown (e.g., mid-setup).
 39``stage`` is a short free-form label like ``"setup"``, ``"predict"``,
 40``"postprocess"``.
 41"""
 42
 43
 44def _sample_meta(sample: Any) -> Dict[str, Any]:
 45    """Extract id/phase/difficulty/scene for attachment to per-sample metrics."""
 46    meta: Dict[str, Any] = {"id": sample.id}
 47    if sample.phase is not None:
 48        meta["phase"] = sample.phase.value
 49    if sample.difficulty is not None:
 50        meta["difficulty"] = sample.difficulty.value
 51    # Scene id is not part of the Sample contract, but the loader passes
 52    # it through via metadata. Manifests write "scene_id"; accept either
 53    # so the cell-log bucketing (keyed on "scene") never silently drops
 54    # rows. See cell_log.cells_from_per_sample.
 55    if sample.metadata:
 56        scene = sample.metadata.get("scene_id") or sample.metadata.get("scene")
 57        if scene is not None:
 58            meta["scene"] = scene
 59    return meta
 60
 61
 62def _align_monocular_depth_predictions_pooled(
 63    predictions: List[DepthPrediction],
 64    samples: List[Any],
 65    *,
 66    mode: str,
 67) -> List[DepthPrediction]:
 68    """Align raw relative-depth predictions once per scene/phase cell.
 69
 70    The manifest may interleave cells, so grouping is by explicit metadata
 71    rather than iteration adjacency. Metric models never enter this helper.
 72    """
 73    groups: Dict[tuple[str, str], List[int]] = defaultdict(list)
 74    for idx, sample in enumerate(samples):
 75        meta = _sample_meta(sample)
 76        scene = meta.get("scene")
 77        phase = meta.get("phase")
 78        if scene is None or phase is None:
 79            raise ModelError(
 80                "Relative Image Depth evaluation requires scene and phase metadata "
 81                "for pooled alignment.",
 82                hint=(
 83                    "Use an RPX manifest whose samples include scene_id and phase; "
 84                    "per-frame alignment is intentionally not used."
 85                ),
 86            )
 87        groups[(str(scene), str(phase))].append(idx)
 88
 89    aligned: List[DepthPrediction | None] = [None] * len(predictions)
 90    for indices in groups.values():
 91        pred_seq = np.stack(
 92            [np.asarray(predictions[i].depth_map, dtype=np.float32) for i in indices]
 93        )
 94        gt_seq = np.stack(
 95            [np.asarray(samples[i].ground_truth.depth_map, dtype=np.float32) for i in indices]
 96        )
 97        aligned_seq = align_pred_to_gt_pooled(pred_seq, gt_seq, mode=mode)
 98        for frame_idx, source_idx in enumerate(indices):
 99            aligned[source_idx] = DepthPrediction(
100                depth_map=np.asarray(aligned_seq[frame_idx], dtype=np.float32)
101            )
102    return [pred for pred in aligned if pred is not None]
103
104
105def _sum_nested_counts(node: Any) -> int:
106    """Walk arbitrarily-nested FlopCounterMode output and sum int leaves.
107
108    Torch changed the shape of ``FlopCounterMode.get_flop_counts`` between
109    versions: older releases returned ``{op_name: int}``, newer releases
110    return ``{module_name: {op_name: int}}`` (often with a top-level
111    "Global" key alongside module-scoped dicts).
112    """
113    if isinstance(node, dict):
114        return sum(_sum_nested_counts(v) for v in node.values())
115    try:
116        return int(node)
117    except (TypeError, ValueError):
118        return 0
119
120
121def _make_cuda_sync():
122    """Return a callable that synchronises CUDA when available, else a no-op.
123
124    Resolved once at runner setup so the per-batch hot loop only pays
125    the function-call overhead (no per-call ``torch.cuda.is_available()``
126    branch). On CPU-only systems or when torch is missing, returns a
127    no-op so the runner stays usable in lightweight test environments.
128    """
129    try:
130        import torch
131    except ImportError:
132        return lambda: None
133    if not torch.cuda.is_available():
134        return lambda: None
135    return torch.cuda.synchronize
136
137
138def _count_flops_of(fn, *args, **kwargs) -> Tuple[Optional[float], Any]:
139    """Invoke ``fn`` inside torch's FlopCounterMode and return ``(g_flops, result)``.
140
141    ``g_flops`` is ``None`` when torch is not installed or the counter
142    errors; in both cases ``fn`` is still invoked so the caller always
143    gets the real result. Note that torch reports **FLOPs, not MACs** —
144    the value is directly comparable to published `G_FLOPs` numbers.
145    """
146    try:
147        from torch.utils.flop_counter import FlopCounterMode
148    except ImportError:
149        return None, fn(*args, **kwargs)
150    try:
151        with FlopCounterMode(display=False) as fcm:
152            result = fn(*args, **kwargs)
153        total_flops = _sum_nested_counts(fcm.get_flop_counts())
154        if total_flops <= 0:
155            return None, result
156        return total_flops / 1e9, result
157    except Exception:
158        return None, fn(*args, **kwargs)
159
160
161class BenchmarkRunner:
162    """Runs a benchmark end-to-end for a given model.
163
164    Basic usage::
165
166        runner = BenchmarkRunner(model=model, dataset=dataset)
167        result = runner.run()
168        print(result.aggregated)
169
170    Phase-stratified usage (requires manifest with ``phase`` / ``difficulty`` fields)::
171
172        runner = BenchmarkRunner(model=model, dataset=dataset)
173        result, report = runner.run_with_report(
174            primary_metric="absrel",
175            model_name="MyDepthModel",
176        )
177
178    The returned ``DeploymentReadinessReport`` carries the per-axis
179    components (scene-change robustness + compute cost). RPX no longer
180    combines those into a single composite — see
181    ``benchmark/README.md`` for the three-axis policy.
182    """
183
184    def __init__(
185        self,
186        model: BenchmarkModel,
187        dataset: RPXDataset,
188        metric_suite: MetricSuite | None = None,
189        call_setup: bool = True,
190    ):
191        self.model = model
192        self.dataset = dataset
193        self.metric_suite = metric_suite or MetricSuite.for_task(model.task)
194        self.call_setup = call_setup
195        self._validate_task_alignment()
196
197    def _task_spec_or_none(self):
198        """Fetch the TaskSpec for the current model, or None if unregistered.
199
200        The runner supports ad-hoc models whose task is not wired into
201        the pluggable task registry (e.g. user-defined custom tasks in
202        a notebook). In that case deployment-readiness hooks simply
203        don't fire and the caller gets back a report without TS / SGC
204        populated.
205        """
206        try:
207            from .tasks.registry import get_task_spec  # noqa: PLC0415 — lazy
208
209            return get_task_spec(self.model.task)
210        except Exception:
211            return None
212
213    def _validate_task_alignment(self) -> None:
214        if not isinstance(self.model.task, TaskType):
215            raise ModelError(
216                "BenchmarkModel must expose `task` as a TaskType enum.",
217                hint=(
218                    "Subclass rpx_benchmark.api.BenchmarkModel or use "
219                    "rpx_benchmark.adapters.BenchmarkableModel which sets "
220                    "task correctly."
221                ),
222            )
223        if self.model.task != self.dataset.task:
224            raise ModelError(
225                f"Task mismatch between model ({self.model.task.value}) "
226                f"and dataset ({self.dataset.task.value}).",
227                hint="Build the dataset and model for the same TaskType.",
228            )
229
230    def run(
231        self,
232        progress: Optional[ProgressCallback] = None,
233    ) -> BenchmarkResult:
234        """Run benchmark and return flat per-sample + aggregated metrics."""
235        if self.call_setup:
236            if progress:
237                progress(0, None, "setup")
238            self.model.setup()
239
240        total = len(self.dataset)
241        per_sample: List[dict] = []
242        all_predictions: List[Any] = []
243        all_samples: List[Any] = []
244        is_relative_depth = (
245            self.model.task is TaskType.MONOCULAR_DEPTH
246            and getattr(self.model, "depth_output_kind", "metric") == "relative"
247        )
248        if progress:
249            progress(0, total, "predict")
250        for batch in self.dataset:
251            predictions = self.model.predict(batch)
252            if len(predictions) != len(batch):
253                raise ModelError(
254                    f"Model returned {len(predictions)} predictions for "
255                    f"a batch of {len(batch)} samples — must return one "
256                    "prediction per sample.",
257                )
258            for sample, pred in zip(batch, predictions, strict=False):
259                validate_prediction(self.model.task, pred, sample)
260                all_predictions.append(pred)
261                all_samples.append(sample)
262                if not is_relative_depth:
263                    metrics = self.metric_suite.evaluate(pred, sample.ground_truth)
264                    per_sample.append(metrics)
265                if progress:
266                    progress(len(all_samples), total, "predict")
267
268        if is_relative_depth:
269            mode = getattr(self.model, "native_alignment", "ls_affine")
270            all_predictions = _align_monocular_depth_predictions_pooled(
271                all_predictions,
272                all_samples,
273                mode=mode,
274            )
275            per_sample = [
276                self.metric_suite.evaluate(pred, sample.ground_truth)
277                for sample, pred in zip(all_samples, all_predictions, strict=True)
278            ]
279
280        return self.metric_suite.build_result(per_sample)
281
282    def run_with_report(
283        self,
284        primary_metric: str,
285        model_name: str = "model",
286        efficiency: EfficiencyMetadata | None = None,
287        compute_ts: bool = True,
288        compute_sgc_flag: bool = True,
289        skip_flops: bool = False,
290        progress: Optional[ProgressCallback] = None,
291    ) -> tuple[BenchmarkResult, DeploymentReadinessReport]:
292        """Run benchmark and compute all per-axis components.
293
294        The returned report carries the scene-change-robustness and
295        compute-cost components. RPX never combines these into a
296        single composite — see ``benchmark/README.md`` for the
297        three-axis policy.
298
299        Args:
300            primary_metric: metric key used for ESD/STR scoring (e.g. "absrel", "miou").
301            model_name: display name for the report.
302            efficiency: pre-computed EfficiencyMetadata (params, FLOPs).
303            compute_ts: whether to compute Temporal Stability (needs sequential frames).
304            compute_sgc_flag: whether to compute SGC (needs both seg + depth predictions).
305            skip_flops: skip the first-batch FLOP counter.  Useful for
306                high-VRAM models on small GPUs where the FLOP-counter
307                dispatch metadata alone may cause OOM even though
308                inference fits.
309
310        Returns:
311            (BenchmarkResult, DeploymentReadinessReport)
312        """
313        if self.call_setup:
314            if progress:
315                progress(0, None, "setup")
316            self.model.setup()
317
318        total = len(self.dataset)
319        per_sample_metrics: List[dict] = []
320        per_sample_phases: List[Phase | None] = []
321        per_sample_difficulties: List[Difficulty | None] = []
322        per_sample_poses: List[Any] = []
323        all_predictions: List[Any] = []
324        all_samples: List[Any] = []
325        spec = self._task_spec_or_none()
326        is_relative_depth = (
327            self.model.task is TaskType.MONOCULAR_DEPTH
328            and getattr(self.model, "depth_output_kind", "metric") == "relative"
329        )
330        incremental_depth_ts = (
331            self.model.task is TaskType.MONOCULAR_DEPTH
332            and compute_ts
333            and spec is not None
334            and spec.temporal_stability_fn is not None
335        )
336        retain_for_hooks = (
337            not incremental_depth_ts
338            and (
339                (compute_ts and spec is not None and spec.temporal_stability_fn is not None)
340                or (
341                    compute_sgc_flag
342                    and spec is not None
343                    and spec.geometric_coherence_fn is not None
344                )
345            )
346        )
347        depth_ts_scores: List[float] = []
348        previous_depth_prediction: Any = None
349        previous_depth_pose: Any = None
350        previous_depth_key: tuple[str, str] | None = None
351        pending_samples: List[Any] = []
352        pending_predictions: List[DepthPrediction] = []
353        pending_key: tuple[str, str] | None = None
354        closed_keys: set[tuple[str, str]] = set()
355
356        def _record_evaluation(sample: Any, pred: Any) -> None:
357            nonlocal previous_depth_key, previous_depth_pose, previous_depth_prediction
358            metrics = self.metric_suite.evaluate(pred, sample.ground_truth)
359            metrics.update(_sample_meta(sample))
360            per_sample_metrics.append(metrics)
361            per_sample_phases.append(sample.phase)
362            per_sample_difficulties.append(sample.difficulty)
363            per_sample_poses.append(sample.camera_pose)
364            if incremental_depth_ts:
365                meta = _sample_meta(sample)
366                current_key = (str(meta.get("scene")), str(meta.get("phase")))
367                if previous_depth_prediction is not None and previous_depth_key == current_key:
368                    pair_result = spec.temporal_stability_fn(
369                        [previous_depth_prediction, pred],
370                        [None, None],
371                        [previous_depth_pose, sample.camera_pose],
372                    )
373                    if pair_result.per_pair:
374                        depth_ts_scores.extend(float(value) for value in pair_result.per_pair)
375                previous_depth_prediction = pred
376                previous_depth_pose = sample.camera_pose
377                previous_depth_key = current_key
378            elif retain_for_hooks:
379                all_predictions.append(pred)
380                all_samples.append(sample)
381
382        def _flush_relative_cell() -> None:
383            if not pending_samples:
384                return
385            mode = getattr(self.model, "native_alignment", "ls_affine")
386            aligned = _align_monocular_depth_predictions_pooled(
387                pending_predictions,
388                pending_samples,
389                mode=mode,
390            )
391            for cell_sample, cell_pred in zip(pending_samples, aligned, strict=True):
392                _record_evaluation(cell_sample, cell_pred)
393            pending_samples.clear()
394            pending_predictions.clear()
395
396        # Latency + memory profiling. Both backends skip gracefully on
397        # devices / libraries that don't support them, so this adds no
398        # hard dependency to the runner.
399        latency = LatencyProfiler(warmup=1)
400        memory = MemoryProfiler().reset()
401        first_batch_flops_g: Optional[float] = None
402        per_sample_latency_values: List[float | None] = []
403
404        # CUDA kernels are async — ``model.predict`` returns when work
405        # is *queued*, not when it *completes*. Without a synchronise
406        # before stopping the timer, latency_ms in the cell log
407        # silently undercounts every GPU-bound model. Sync once before
408        # t0 (so kernels from outside the loop don't bleed into our
409        # measurement) and once after predict (so we measure to GPU
410        # completion). On CPU-only runs this is a no-op.
411        _cuda_sync = _make_cuda_sync()
412
413        if progress:
414            progress(0, total, "predict")
415
416        first_batch = True
417        prediction_count = 0
418        for batch in self.dataset:
419            _cuda_sync()
420            t0 = time.perf_counter()
421            if first_batch and not skip_flops:
422                # Count FLOPs on a single sample to avoid scaling
423                # the FLOP-counter dispatch overhead by batch size.
424                # FLOPs are per-sample, so counting on one is enough.
425                single = [batch[0]]
426                flops_g, _ = _count_flops_of(self.model.predict, single)
427                first_batch_flops_g = flops_g  # already per-sample
428                # Now run the full batch normally (timed)
429                predictions = self.model.predict(batch)
430                first_batch = False
431            elif first_batch and skip_flops:
432                predictions = self.model.predict(batch)
433                first_batch = False
434            else:
435                predictions = self.model.predict(batch)
436            _cuda_sync()
437            batch_seconds = time.perf_counter() - t0
438
439            # Saving compressed predictions is deliberately outside the
440            # inference timer. Models that do not implement this optional
441            # hook retain the historical behaviour.
442            persist = getattr(self.model, "persist_predictions", None)
443            if callable(persist):
444                persist(batch, predictions)
445
446            if len(predictions) != len(batch):
447                raise ModelError(
448                    f"Model returned {len(predictions)} predictions for "
449                    f"a batch of {len(batch)} samples — must return one "
450                    "prediction per sample.",
451                )
452            cache_hits = getattr(self.model, "last_cache_hits", None)
453            if isinstance(cache_hits, list) and len(cache_hits) == len(batch):
454                inferred_count = sum(not bool(hit) for hit in cache_hits)
455                latency.add_batch_seconds(batch_seconds, inferred_count)
456                inferred_ms = (
457                    batch_seconds * 1000.0 / inferred_count if inferred_count else None
458                )
459                per_sample_latency_values.extend(
460                    None if hit else inferred_ms for hit in cache_hits
461                )
462            else:
463                latency.add_batch_seconds(batch_seconds, len(batch))
464                per_sample_latency_values.extend(
465                    [batch_seconds * 1000.0 / len(batch)] * len(batch)
466                )
467
468            for sample, pred in zip(batch, predictions, strict=False):
469                validate_prediction(self.model.task, pred, sample)
470                if is_relative_depth:
471                    meta = _sample_meta(sample)
472                    if meta.get("scene") is None or meta.get("phase") is None:
473                        raise ModelError(
474                            "Relative Image Depth evaluation requires scene and phase "
475                            "metadata for pooled alignment.",
476                            hint="Use an RPX manifest with scene_id and phase fields.",
477                        )
478                    key = (str(meta["scene"]), str(meta["phase"]))
479                    if pending_key is None:
480                        pending_key = key
481                    elif key != pending_key:
482                        _flush_relative_cell()
483                        closed_keys.add(pending_key)
484                        if key in closed_keys:
485                            raise ModelError(
486                                f"Manifest cell {key!r} is non-contiguous; pooled alignment "
487                                "cannot be streamed safely.",
488                                hint="Sort manifest samples by scene_id, phase and frame_idx.",
489                            )
490                        pending_key = key
491                    pending_samples.append(sample)
492                    pending_predictions.append(pred)
493                else:
494                    _record_evaluation(sample, pred)
495                prediction_count += 1
496                if progress:
497                    progress(prediction_count, total, "predict")
498
499        if is_relative_depth:
500            log.info(
501                "model %s is relative-depth: applying %s once per streamed scene/phase cell",
502                model_name,
503                getattr(self.model, "native_alignment", "ls_affine"),
504            )
505            _flush_relative_cell()
506
507        # Attach per-sample latency_ms so downstream analysis can group
508        # timings with the metric values. LatencyProfiler.samples_ms()
509        # produces one entry per sample (batch timing amortised evenly
510        # across the batch's samples).
511        for row, lat_ms in zip(
512            per_sample_metrics,
513            per_sample_latency_values,
514            strict=False,
515        ):
516            if lat_ms is not None:
517                row["latency_ms"] = float(lat_ms)
518
519        result = self.metric_suite.build_result(per_sample_metrics)
520
521        latency_percentiles = latency.percentiles()
522        latency_ms = latency_percentiles["p50_ms"]  # back-compat: median
523        memory_peaks = memory.peaks()
524
525        # --- Weighted Phase Score + STR ---
526        wps = compute_weighted_phase_score(
527            per_sample_metrics=per_sample_metrics,
528            per_sample_phases=per_sample_phases,
529            per_sample_difficulties=per_sample_difficulties,
530            metric_key=primary_metric,
531        )
532
533        str_result = compute_str(
534            {
535                Phase.CLUTTER: wps.s_clutter,
536                Phase.INTERACTION: wps.s_interaction,
537                Phase.CLEAN: wps.s_clean,
538            }
539        )
540
541        # --- Deployment-readiness hooks dispatched via TaskSpec ---
542        # The runner no longer branches on task identity. Each task
543        # that wants Temporal Stability or Stack Geometric Coherence
544        # registers its own implementation on its TaskSpec; tasks
545        # without a registered hook silently skip the computation.
546        ts_result: TemporalStabilityResult | None = None
547        if incremental_depth_ts:
548            ts_result = TemporalStabilityResult(
549                ts_score=float(np.mean(depth_ts_scores)) if depth_ts_scores else 1.0,
550                num_pairs=len(depth_ts_scores),
551                per_pair=depth_ts_scores,
552            )
553        elif (
554            compute_ts
555            and len(all_predictions) >= 2
556            and spec is not None
557            and spec.temporal_stability_fn is not None
558        ):
559            ts_result = spec.temporal_stability_fn(
560                all_predictions,
561                all_samples,
562                per_sample_poses,
563            )
564
565        sgc_result: StackGeometricCoherenceResult | None = None
566        if compute_sgc_flag and spec is not None and spec.geometric_coherence_fn is not None:
567            sgc_result = spec.geometric_coherence_fn(all_predictions, all_samples)
568
569        # Merge counted FLOPs + measured latency + memory into the
570        # passed-in efficiency object (caller usually pre-computed
571        # params_m). A fresh object is created when the caller did not
572        # provide one so downstream reporting always has a target to
573        # write measured values onto.
574        eff = efficiency or EfficiencyMetadata()
575        eff.flops_g = eff.flops_g or first_batch_flops_g
576        eff.latency_ms_per_sample = latency_ms
577        eff.latency_p50_ms = latency_percentiles["p50_ms"]
578        eff.latency_p95_ms = latency_percentiles["p95_ms"]
579        eff.latency_p99_ms = latency_percentiles["p99_ms"]
580        eff.peak_cpu_mb = memory_peaks["cpu_mb"]
581        eff.peak_cuda_mb = memory_peaks["cuda_mb"]
582        eff.peak_mps_mb = memory_peaks["mps_mb"]
583
584        # Peak resident memory across the three possible backends
585        # (CPU / CUDA / MPS). We pick the max so the ERS has a single
586        # "worst-case device memory" number regardless of where the
587        # model actually ran.
588        peak_mb_vals = [v for v in memory_peaks.values() if v is not None]
589        peak_memory_mb = max(peak_mb_vals) if peak_mb_vals else None
590
591        # Ensure derived Tier-1 fields (MACs, traffic, AI) are populated
592        # now that flops_g may have been filled by the FLOP counter above.
593        eff.derive_tier1()
594
595        # Compute Tier-2 roofline bounds if we have enough data
596        if eff.flops_g is not None and eff.roofline is None:
597            eff.compute_roofline()
598
599        # Serialise roofline bounds for the report
600        roofline_dict = None
601        if eff.roofline:
602            roofline_dict = {
603                name: {
604                    "compute_ms": b.compute_ms,
605                    "memory_ms": b.memory_ms,
606                    "latency_ms": b.latency_ms,
607                    "bottleneck": b.bottleneck,
608                }
609                for name, b in eff.roofline.items()
610            }
611
612        # Serialise system card if present
613        system_card_dict = eff.system_card.to_dict() if eff.system_card else None
614
615        report = DeploymentReadinessReport(
616            task=self.model.task.value,
617            model_name=model_name,
618            weighted_phase_score=wps,
619            temporal_stability=ts_result,
620            state_transition=str_result,
621            geometric_coherence=sgc_result,
622            # Tier 1
623            params_m=eff.params_m,
624            flops_g=eff.flops_g,
625            macs_g=eff.macs_g,
626            actmem_gb_fp16=eff.actmem_gb_fp16,
627            memory_traffic_gb=eff.memory_traffic_gb,
628            arithmetic_intensity=eff.arithmetic_intensity,
629            # Tier 2
630            roofline=roofline_dict,
631            # Tier 3
632            latency_ms_per_sample=eff.latency_ms_per_sample,
633            peak_memory_mb=peak_memory_mb,
634            system_card=system_card_dict,
635        )
636
637        # Build an OperatingPoint for this run (the precision × accuracy ×
638        # cost triple). RPX reports operating points but never combines
639        # them into a composite score — see benchmark/README.md.
640        higher_is_better = bool(getattr(spec, "higher_is_better", True))
641        from .deployment import OperatingPoint  # noqa: PLC0415
642
643        if wps is not None and eff.flops_g is not None:
644            # Precision: prefer the model's declared native_precision,
645            # fall back to the system card, default to fp32.
646            precision = getattr(self.model, "native_precision", None) or (
647                eff.system_card.get("precision", "fp32")
648                if isinstance(eff.system_card, dict)
649                else getattr(eff.system_card, "precision", "fp32")
650                if eff.system_card
651                else "fp32"
652            )
653            str_val = 0.0
654            if str_result is not None:
655                str_val = str_result.str_c_to_i or 0.0
656            report.operating_point = OperatingPoint(
657                precision=precision,
658                task_metric=wps.s_overall,
659                task_metric_name=primary_metric,
660                higher_is_better=higher_is_better,
661                str_score=str_val,
662                flops_g=eff.flops_g,
663                params_m=eff.params_m or 0.0,
664                memory_traffic_gb=eff.memory_traffic_gb,
665            )
666
667        return result, report