From 21b0f54bfc2ad4503d961e164101677b0e801ec8 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 13 Aug 2026 15:41:13 +0200 Subject: [PATCH 1/3] extract and passthrough context that doesnt change at runtime once per run and remove ambiguous kwargs --- luxonis_eval/core/context.py | 32 ++++++++++ luxonis_eval/core/core.py | 53 ++++++++-------- luxonis_eval/core/runtime.py | 47 ++++---------- luxonis_eval/metrics/base_metric.py | 19 +++++- luxonis_eval/metrics/bbox_map.py | 22 +++---- luxonis_eval/metrics/dice_coef.py | 10 ++- luxonis_eval/metrics/f1_score.py | 8 +-- luxonis_eval/metrics/jaccard_index.py | 8 +-- luxonis_eval/metrics/keypoint_map.py | 10 +-- luxonis_eval/metrics/mIoU.py | 11 ++-- luxonis_eval/metrics/mask_map.py | 14 ++--- luxonis_eval/metrics/topk_accuracy.py | 15 ++--- luxonis_eval/parsers/base_parser.py | 24 ++++--- luxonis_eval/parsers/classification.py | 26 ++------ luxonis_eval/parsers/segmentation.py | 18 ++---- luxonis_eval/parsers/yolo.py | 70 ++++++++++----------- luxonis_eval/visualizers/base_visualizer.py | 17 ++++- 17 files changed, 203 insertions(+), 201 deletions(-) create mode 100644 luxonis_eval/core/context.py diff --git a/luxonis_eval/core/context.py b/luxonis_eval/core/context.py new file mode 100644 index 0000000..e6c846c --- /dev/null +++ b/luxonis_eval/core/context.py @@ -0,0 +1,32 @@ +from collections.abc import Callable +from dataclasses import dataclass + +import numpy as np + +from luxonis_eval.engines.base_engine import ModelSpec + +TargetConverter = Callable[ + [np.ndarray, int, int], + tuple[np.ndarray, np.ndarray], +] + + +@dataclass(frozen=True, slots=True) +class EvalContext: + """Static runtime metadata shared across one evaluation run.""" + + model_spec: ModelSpec + class_map: dict[int, str] + target_class_map: dict[int, str] + class_index_map: dict[int, int] | None + category_ids: tuple[int, ...] + target_background_index: int | None + target_converter: TargetConverter + + @property + def width(self) -> int: + return self.model_spec.width + + @property + def height(self) -> int: + return self.model_spec.height diff --git a/luxonis_eval/core/core.py b/luxonis_eval/core/core.py index b9f77d6..b34ddbc 100644 --- a/luxonis_eval/core/core.py +++ b/luxonis_eval/core/core.py @@ -8,6 +8,7 @@ from luxonis_ml.typing import Params, PathType from luxonis_eval.config import EvalConfig, EvaluatorConfig +from luxonis_eval.core.context import EvalContext from luxonis_eval.core.factories import ( create_engine, create_loader, @@ -22,7 +23,7 @@ get_model_name, ) from luxonis_eval.core.runtime import ( - build_metric_contexts, + build_eval_context, normalize_target, resolve_class_mapping, select_evaluator_outputs, @@ -103,13 +104,13 @@ def setup(self) -> None: loader_params=self.cfg.pipeline.loader.params, loader_task_name=self.loader_task_name, ) - self.metric_contexts = build_metric_contexts( - self.evaluator_cfg, + self.eval_context = build_eval_context( model_spec=self.model_spec, ldf_class_map=self.ldf_class_map, class_map=self.class_map, class_index_map=self.class_index_map, ) + self._attach_eval_context() self._sanity_check_pipeline() except Exception: try: @@ -147,7 +148,6 @@ def _run_evaluators(self) -> EvaluationResult: assert self.parser is not None assert self.throughput_metric is not None assert self.evaluator_cfg is not None - assert self.model_spec is not None with self._progress( f"Running {engine_name.upper()} inference ({model_name})...", @@ -170,21 +170,15 @@ def _run_evaluators(self) -> EvaluationResult: select_evaluator_outputs( raw_output, self.evaluator_cfg.outputs, - ), - model_spec=self.model_spec, - class_map=self.class_map, - **self.evaluator_cfg.parser.params, + ) ) parsing_elapsed = time.perf_counter() - parsing_t0 metric_update_t0 = time.perf_counter() - for metric, metric_ctx in zip( - self.metrics, self.metric_contexts, strict=True - ): + for metric in self.metrics: metric.update( predictions=predictions, target=target, - **metric_ctx, ) metric_update_elapsed = ( time.perf_counter() - metric_update_t0 @@ -201,16 +195,12 @@ def _run_evaluators(self) -> EvaluationResult: for visualizer_cfg in self.evaluator_cfg.visualizers if visualizer_cfg.active ] - for visualizer, visualizer_cfg in zip( + for visualizer, _visualizer_cfg in zip( self.visualizers, active_visualizer_cfgs, strict=True, ): - visualizer.visualize( - predictions, - self.engine.vis_frame(), - **visualizer_cfg.params, - ) + visualizer.visualize(predictions, self.engine.vis_frame()) progress.update(advance=1) metric_compute_t0 = time.perf_counter() @@ -258,7 +248,6 @@ def _sanity_check_pipeline(self) -> None: assert self.engine is not None assert self.parser is not None assert self.evaluator_cfg is not None - assert self.model_spec is not None if len(self.loader) == 0: raise ValueError( @@ -276,15 +265,10 @@ def _sanity_check_pipeline(self) -> None: ) raw_output = self.engine.infer_once(img) predictions = self.parser.parse( - select_evaluator_outputs(raw_output, self.evaluator_cfg.outputs), - model_spec=self.model_spec, - class_map=self.class_map, - **self.evaluator_cfg.parser.params, + select_evaluator_outputs(raw_output, self.evaluator_cfg.outputs) ) - for metric, metric_ctx in zip( - self.metrics, self.metric_contexts, strict=True - ): + for metric in self.metrics: missing = set(metric.required_target_keys()) - set(target) if missing: raise ValueError( @@ -296,7 +280,6 @@ def _sanity_check_pipeline(self) -> None: metric.update( predictions=predictions, target=target, - **metric_ctx, ) metric.compute() metric.reset() @@ -310,13 +293,27 @@ def _clear_runtime_fields(self) -> None: self.visualizers: list[BaseVisualizer] = [] self.evaluator_cfg: EvaluatorConfig | None = None + self.eval_context: EvalContext | None = None self.model_spec: ModelSpec | None = None self.loader_task_name: str | None = None self.ldf_class_map: dict[int, str] = {} self.class_map: dict[int, str] = {} self.class_index_map: dict[int, int] | None = None - self.metric_contexts: list[dict[str, Any]] = [] + + def _attach_eval_context(self) -> None: + if self.eval_context is None: + raise RuntimeError( + "Evaluation context is unavailable before setup." + ) + if self.parser is None: + raise RuntimeError("Parser is unavailable before setup.") + + self.parser.attach_context(self.eval_context) + for metric in self.metrics: + metric.attach_context(self.eval_context) + for visualizer in self.visualizers: + visualizer.attach_context(self.eval_context) def _require_setup(self) -> None: if not self._is_setup: diff --git a/luxonis_eval/core/runtime.py b/luxonis_eval/core/runtime.py index 7b7148d..d26172f 100644 --- a/luxonis_eval/core/runtime.py +++ b/luxonis_eval/core/runtime.py @@ -9,6 +9,7 @@ from luxonis_ml.typing import Params from luxonis_eval.config import EvaluatorConfig +from luxonis_eval.core.context import EvalContext from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput from luxonis_eval.loaders.base_loader import BaseEvalLoader @@ -40,46 +41,22 @@ def resolve_class_mapping( return loader.get_class_mapping(**loader_params) -def build_metric_contexts( - evaluator_cfg: EvaluatorConfig, +def build_eval_context( model_spec: ModelSpec, ldf_class_map: dict[int, str], class_map: dict[int, str], class_index_map: dict[int, int] | None, -) -> list[dict[str, Any]]: - return [ - get_metric_ctx( - base_ctx=metric_cfg.params, - width=model_spec.width, - height=model_spec.height, - ldf_class_map=ldf_class_map, - class_map=class_map, - class_index_map=class_index_map, - ) - for metric_cfg in evaluator_cfg.metrics - ] - - -def get_metric_ctx(base_ctx: dict[str, Any], **kwargs: Any) -> dict[str, Any]: - class_index_map = kwargs.get("class_index_map") - class_map = kwargs.get("class_map") or {} - ldf_class_map = kwargs.get("ldf_class_map") or {} - width = kwargs.get("width", -1) - height = kwargs.get("height", -1) - +) -> EvalContext: ldf_name_to_idx = {v: k for k, v in ldf_class_map.items()} - - return { - **base_ctx, - "class_map": class_map, - "class_index_map": class_index_map, - "width": width, - "height": height, - "category_ids": sorted(class_map.keys()), - "target_converter": normalized_xywh_to_coco_xywh, - "target_bg": ldf_name_to_idx.get("background"), - "target_class_map": ldf_class_map, - } + return EvalContext( + model_spec=model_spec, + class_map=class_map, + target_class_map=ldf_class_map, + class_index_map=class_index_map, + category_ids=tuple(sorted(class_map.keys())), + target_background_index=ldf_name_to_idx.get("background"), + target_converter=normalized_xywh_to_coco_xywh, + ) def select_evaluator_outputs( diff --git a/luxonis_eval/metrics/base_metric.py b/luxonis_eval/metrics/base_metric.py index 4a6a1d8..634410c 100644 --- a/luxonis_eval/metrics/base_metric.py +++ b/luxonis_eval/metrics/base_metric.py @@ -4,6 +4,7 @@ import numpy as np from luxonis_ml.utils.registry import AutoRegisterMeta +from luxonis_eval.core.context import EvalContext from luxonis_eval.parsers.predictions import Prediction from luxonis_eval.registry import METRICS_REGISTRY @@ -23,8 +24,23 @@ def __init__(self, **kwargs: Any) -> None: **kwargs : Any Metric basic configuration. """ + del kwargs + self._context: EvalContext | None = None self.reset() + def attach_context(self, context: EvalContext) -> None: + """Attach evaluation runtime metadata after setup.""" + self._context = context + + def require_context(self) -> EvalContext: + """Return the attached evaluation context.""" + if self._context is None: + raise RuntimeError( + f"{type(self).__name__} is missing evaluation context. " + "Call attach_context() during setup before update()." + ) + return self._context + @abstractmethod def required_target_keys(self) -> list[str]: """Return the ground-truth keys required by the metric.""" @@ -40,7 +56,6 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: """Update the metric with predictions and ground truths. @@ -50,8 +65,6 @@ def update( Structured model predictions. target : dict[str, np.ndarray] Ground-truth data. - **kwargs : Any - Additional context. """ ... diff --git a/luxonis_eval/metrics/bbox_map.py b/luxonis_eval/metrics/bbox_map.py index 29ade93..3904f90 100644 --- a/luxonis_eval/metrics/bbox_map.py +++ b/luxonis_eval/metrics/bbox_map.py @@ -46,7 +46,6 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: """Update internal metric state. @@ -56,22 +55,17 @@ def update( Structured detection predictions. target : dict[str, np.ndarray] Ground-truth data. - **kwargs : Any - Additional context. """ + context = self.require_context() detections = predictions.require_detections() target_boxes = target[self.required_target_keys()[0]] - width = int(kwargs["width"]) - height = int(kwargs["height"]) - - class_map: dict[int, str] = kwargs.get("class_map", {}) - category_ids: Sequence[int] | None = kwargs.get("category_ids") - class_index_map = kwargs.get("class_index_map") - target_converter = kwargs.get("target_converter") - if target_converter is None: - raise ValueError( - "BboxMeanAveragePrecision requires target_converter in ctx." - ) + width = context.width + height = context.height + + class_map = context.class_map + category_ids: Sequence[int] = context.category_ids + class_index_map = context.class_index_map + target_converter = context.target_converter self._store.init_categories_once( class_map=class_map, category_ids=category_ids diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index fda047a..1c66a3a 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -64,7 +64,6 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: """Update internal metric state. @@ -74,18 +73,17 @@ def update( Structured segmentation predictions. target : dict[str, np.ndarray] Ground-truth labels. - **kwargs : Any - Additional context. """ + context = self.require_context() if self.target_class_map is None: - self.target_class_map = kwargs.get("target_class_map", {}) + self.target_class_map = context.target_class_map prepared = prepare_segmentation_metric_inputs( predictions, target, include_background=self.include_background, target_key=self.required_target_keys()[0], - target_bg=kwargs.get("target_bg"), - class_index_map=kwargs.get("class_index_map"), + target_bg=context.target_background_index, + class_index_map=context.class_index_map, ) self.metric.update(prepared.pred_tensor, prepared.target_tensor) diff --git a/luxonis_eval/metrics/f1_score.py b/luxonis_eval/metrics/f1_score.py index af8c9a1..e7ba10b 100644 --- a/luxonis_eval/metrics/f1_score.py +++ b/luxonis_eval/metrics/f1_score.py @@ -44,17 +44,17 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: + context = self.require_context() if self.target_class_map is None: - self.target_class_map = kwargs.get("target_class_map", {}) + self.target_class_map = context.target_class_map prepared = prepare_segmentation_metric_inputs( predictions, target, include_background=self.include_background, target_key=self.required_target_keys()[0], - target_bg=kwargs.get("target_bg"), - class_index_map=kwargs.get("class_index_map"), + target_bg=context.target_background_index, + class_index_map=context.class_index_map, ) if self.metric is None: diff --git a/luxonis_eval/metrics/jaccard_index.py b/luxonis_eval/metrics/jaccard_index.py index af04b17..0b2fe83 100644 --- a/luxonis_eval/metrics/jaccard_index.py +++ b/luxonis_eval/metrics/jaccard_index.py @@ -43,17 +43,17 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: + context = self.require_context() if self.target_class_map is None: - self.target_class_map = kwargs.get("target_class_map", {}) + self.target_class_map = context.target_class_map prepared = prepare_segmentation_metric_inputs( predictions, target, include_background=self.include_background, target_key=self.required_target_keys()[0], - target_bg=kwargs.get("target_bg"), - class_index_map=kwargs.get("class_index_map"), + target_bg=context.target_background_index, + class_index_map=context.class_index_map, ) if self.metric is None: diff --git a/luxonis_eval/metrics/keypoint_map.py b/luxonis_eval/metrics/keypoint_map.py index e6b2003..adca24e 100644 --- a/luxonis_eval/metrics/keypoint_map.py +++ b/luxonis_eval/metrics/keypoint_map.py @@ -66,16 +66,16 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: + context = self.require_context() detections = predictions.require_detections() target_boxes = target[self.required_target_keys()[0]] target_kpts = target[self.required_target_keys()[1]] - width = int(kwargs["width"]) - height = int(kwargs["height"]) + width = context.width + height = context.height - class_index_map = kwargs.get("class_index_map") - target_converter = kwargs.get("target_converter") + class_index_map = context.class_index_map + target_converter = context.target_converter pred_boxes_xyxy: list[list[float]] = [] pred_scores: list[float] = [] diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index a689158..bc98bce 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -64,7 +64,6 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: """Update internal metric state. @@ -74,19 +73,17 @@ def update( Structured segmentation predictions. target : dict[str, np.ndarray] Ground-truth labels. - **kwargs : Any - Additional context. """ - # Retrieve additional metric-specific options + context = self.require_context() if self.target_class_map is None: - self.target_class_map = kwargs.get("target_class_map", {}) + self.target_class_map = context.target_class_map prepared = prepare_segmentation_metric_inputs( predictions, target, include_background=self.include_background, target_key=self.required_target_keys()[0], - target_bg=kwargs.get("target_bg"), - class_index_map=kwargs.get("class_index_map"), + target_bg=context.target_background_index, + class_index_map=context.class_index_map, ) self.metric.update(prepared.pred_tensor, prepared.target_tensor) diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index fb55cbd..aed8ba8 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -56,7 +56,6 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: """Update internal metric state. @@ -66,18 +65,17 @@ def update( Structured instance-segmentation predictions. target : dict[str, np.ndarray] Ground-truth data. - **kwargs : Any - Additional context. """ + context = self.require_context() target_boxes = target[self.required_target_keys()[0]] target_masks = target[self.required_target_keys()[1]] - width = int(kwargs["width"]) - height = int(kwargs["height"]) + width = context.width + height = context.height - category_ids: Sequence[int] | None = kwargs.get("category_ids") - class_index_map = kwargs.get("class_index_map") - target_converter = kwargs.get("target_converter") + category_ids: Sequence[int] = context.category_ids + class_index_map = context.class_index_map + target_converter = context.target_converter target_classes, target_boxes_xywh = target_converter( target_boxes, width, height diff --git a/luxonis_eval/metrics/topk_accuracy.py b/luxonis_eval/metrics/topk_accuracy.py index e076028..f725ba7 100644 --- a/luxonis_eval/metrics/topk_accuracy.py +++ b/luxonis_eval/metrics/topk_accuracy.py @@ -42,7 +42,6 @@ def update( self, predictions: Prediction, target: dict[str, np.ndarray], - **kwargs: Any, ) -> None: """Update internal metric state. @@ -52,15 +51,11 @@ def update( Structured classification predictions. target : dict[str, np.ndarray] Ground-truth labels. - **kwargs : Any - Additional context. """ cls_target = target[self.required_target_keys()[0]] - class_index_map = kwargs.get("class_index_map") - class_map = kwargs.get("class_map", {}) - class_map = {v: k for k, v in class_map.items()} - - topk = tuple(kwargs.get("topk", self.topk)) + context = self.require_context() + class_index_map = context.class_index_map + class_map = {v: k for k, v in context.class_map.items()} pred_classes = predictions.require_classification().classes tgt = np.asarray(cls_target) @@ -71,10 +66,10 @@ def update( if class_index_map is not None: target_idx = int(class_index_map[target_idx]) - max_k = max(topk) + max_k = max(self.topk) top_idx = [class_map[pred_classes[i]] for i in range(max_k)] # type: ignore - for k in topk: + for k in self.topk: if k not in self.correct_at_k: self.correct_at_k[k] = 0 if target_idx in top_idx[:k]: diff --git a/luxonis_eval/parsers/base_parser.py b/luxonis_eval/parsers/base_parser.py index 0ef94df..da091ab 100644 --- a/luxonis_eval/parsers/base_parser.py +++ b/luxonis_eval/parsers/base_parser.py @@ -3,7 +3,7 @@ from luxonis_ml.utils.registry import AutoRegisterMeta -from luxonis_eval.engines.base_engine import ModelSpec +from luxonis_eval.core.context import EvalContext from luxonis_eval.engines.io import EngineOutput from luxonis_eval.parsers.predictions import Prediction from luxonis_eval.registry import PARSERS_REGISTRY @@ -24,13 +24,23 @@ def __init__(self, **kwargs: Any) -> None: **kwargs : Any Parser basic configuration. """ + del kwargs + self._context: EvalContext | None = None + + def attach_context(self, context: EvalContext) -> None: + """Attach evaluation runtime metadata after setup.""" + self._context = context + + def require_context(self) -> EvalContext: + """Return the attached evaluation context.""" + if self._context is None: + raise RuntimeError( + f"{type(self).__name__} is missing evaluation context. " + "Call attach_context() during setup before parse()." + ) + return self._context @abstractmethod - def parse( - self, - output: EngineOutput, - model_spec: ModelSpec, - **kwargs: Any, - ) -> Prediction: + def parse(self, output: EngineOutput) -> Prediction: """Parse raw backend output into a structured prediction.""" ... diff --git a/luxonis_eval/parsers/classification.py b/luxonis_eval/parsers/classification.py index ed7ff70..b5551f6 100644 --- a/luxonis_eval/parsers/classification.py +++ b/luxonis_eval/parsers/classification.py @@ -6,7 +6,6 @@ ClassificationParser as DepthAINodesClassificationParser, ) -from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput from luxonis_eval.parsers.base_parser import BaseParser from luxonis_eval.parsers.predictions import Prediction @@ -16,39 +15,24 @@ class ClassificationParser(BaseParser): """Parser for classification model outputs.""" - def __init__(self, **kwargs: Any) -> None: + def __init__(self, apply_softmax: bool = False, **kwargs: Any) -> None: """Initialize the classification parser.""" super().__init__(**kwargs) + self.apply_softmax = apply_softmax - def parse( - self, - output: EngineOutput, - model_spec: ModelSpec, - *, - class_map: dict[int, str], - apply_softmax: bool = False, - **kwargs: Any, - ) -> Prediction: + def parse(self, output: EngineOutput) -> Prediction: """Parse backend output into class scores. Parameters ---------- output : EngineOutput Engine-normalized inference output. - model_spec : ModelSpec - Resolved model IO metadata. - apply_softmax : bool, default=False - Whether to apply softmax to the output scores. - **kwargs : Any - Additional parser arguments. - Returns ------- Prediction Structured classification scores. """ - del model_spec, kwargs - classes = ordered_class_names(class_map) + classes = ordered_class_names(self.require_context().class_map) _, scores = output.get_first() scores = np.asarray(scores, dtype=np.float64).flatten() if scores.size == 0: @@ -60,7 +44,7 @@ def parse( "post-processing." ) - if apply_softmax: + if self.apply_softmax: # Subtract the largest value first so softmax stays numerically stable. scores = scores - np.max(scores) scores = DepthAINodesClassificationParser.compute( diff --git a/luxonis_eval/parsers/segmentation.py b/luxonis_eval/parsers/segmentation.py index c744b35..fe886ac 100644 --- a/luxonis_eval/parsers/segmentation.py +++ b/luxonis_eval/parsers/segmentation.py @@ -7,7 +7,6 @@ SegmentationParser as DepthAINodesSegmentationParser, ) -from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput from luxonis_eval.parsers.predictions import Prediction @@ -17,24 +16,19 @@ class SegmentationParser(BaseParser): """Parser for semantic segmentation model outputs.""" - def __init__(self, **kwargs: Any) -> None: + def __init__( + self, classes_in_one_layer: bool = False, **kwargs: Any + ) -> None: """Initialize the segmentation parser.""" super().__init__(**kwargs) + self.classes_in_one_layer = classes_in_one_layer - def parse( - self, - output: EngineOutput, - model_spec: ModelSpec, - *, - classes_in_one_layer: bool = False, - **kwargs: Any, - ) -> Prediction: + def parse(self, output: EngineOutput) -> Prediction: """Parse backend output into segmentation predictions.""" - del model_spec, kwargs _, segmentation_mask = output.get_first() class_map = DepthAINodesSegmentationParser.compute( np.asarray(segmentation_mask), - classes_in_one_layer=classes_in_one_layer, + classes_in_one_layer=self.classes_in_one_layer, ) return Prediction( diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index a78b73b..bd5983d 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -7,7 +7,6 @@ YOLOExtendedParser as DepthAINodesYOLOExtendedParser, ) -from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput from .base_parser import BaseParser @@ -26,43 +25,42 @@ class YOLOExtendedParser(BaseParser): _SEG_MODE = 2 def __init__(self, **kwargs: Any) -> None: + self.subtype: str = kwargs.pop("subtype") + self.n_classes: int | None = kwargs.pop("n_classes", None) + self.anchors: list[list[list[float]]] | None = kwargs.pop( + "anchors", None + ) + self.strides: list[int] | None = kwargs.pop("strides", None) + self.conf_threshold: float = kwargs.pop("conf_threshold", 0.5) + self.iou_threshold: float = kwargs.pop("iou_threshold", 0.5) + self.n_keypoints: int | None = kwargs.pop("n_keypoints", None) + self.mask_conf: float = kwargs.pop("mask_conf", 0.5) + self.max_det: int = kwargs.pop("max_det", 300) + self.keypoint_label_names: list[str] | None = kwargs.pop( + "keypoint_label_names", None + ) + self.keypoint_edges: list[tuple[int, int]] | None = kwargs.pop( + "keypoint_edges", None + ) super().__init__(**kwargs) - def parse( - self, - output: EngineOutput, - model_spec: ModelSpec, - *, - class_map: dict[int, str], - subtype: str, - n_classes: int | None = None, - anchors: list[list[list[float]]] | None = None, - strides: list[int] | tuple[int, ...] | None = None, - conf_threshold: float = 0.5, - iou_threshold: float = 0.5, - n_keypoints: int | None = None, - mask_conf: float = 0.5, - max_det: int = 300, - keypoint_label_names: list[str] | None = None, - keypoint_edges: list[tuple[int, int]] | None = None, - **kwargs: Any, - ) -> Prediction: - del kwargs + def parse(self, output: EngineOutput) -> Prediction: + context = self.require_context() compute_inputs = build_yolo_compute_inputs( output, - model_spec=model_spec, - class_map=class_map, - subtype=subtype, - n_classes=n_classes, - anchors=anchors, - strides=list(strides) if strides is not None else None, - conf_threshold=conf_threshold, - iou_threshold=iou_threshold, - max_det=max_det, - n_keypoints=n_keypoints, - mask_conf=mask_conf, - keypoint_label_names=keypoint_label_names, - keypoint_edges=keypoint_edges, + model_spec=context.model_spec, + class_map=context.class_map, + subtype=self.subtype, + n_classes=self.n_classes, + anchors=self.anchors, + strides=list(self.strides) if self.strides is not None else None, + conf_threshold=self.conf_threshold, + iou_threshold=self.iou_threshold, + max_det=self.max_det, + n_keypoints=self.n_keypoints, + mask_conf=self.mask_conf, + keypoint_label_names=self.keypoint_label_names, + keypoint_edges=self.keypoint_edges, ) raw_outputs_values = None if compute_inputs.masks_outputs_values is not None: @@ -83,11 +81,11 @@ def parse( keypoints_scores=payload["keypoints_scores"], keypoint_label_names=payload.get( "keypoint_label_names", - keypoint_label_names, + self.keypoint_label_names, ), keypoint_edges=payload.get( "keypoint_edges", - keypoint_edges, + self.keypoint_edges, ), ) ) diff --git a/luxonis_eval/visualizers/base_visualizer.py b/luxonis_eval/visualizers/base_visualizer.py index 0b1b705..fb12116 100644 --- a/luxonis_eval/visualizers/base_visualizer.py +++ b/luxonis_eval/visualizers/base_visualizer.py @@ -4,6 +4,7 @@ import numpy as np from luxonis_ml.utils.registry import AutoRegisterMeta +from luxonis_eval.core.context import EvalContext from luxonis_eval.parsers.predictions import Prediction from luxonis_eval.registry import VISUALIZERS_REGISTRY @@ -23,13 +24,27 @@ def __init__(self, **kwargs: Any) -> None: **kwargs : Any Visualizer basic configuration. """ + del kwargs + self._context: EvalContext | None = None + + def attach_context(self, context: EvalContext) -> None: + """Attach evaluation runtime metadata after setup.""" + self._context = context + + def require_context(self) -> EvalContext: + """Return the attached evaluation context.""" + if self._context is None: + raise RuntimeError( + f"{type(self).__name__} is missing evaluation context. " + "Call attach_context() during setup before visualize()." + ) + return self._context @abstractmethod def visualize( self, predictions: Prediction, vis_frame: np.ndarray, - **kwargs: Any, ) -> None: """Visualize the evaluation results.""" ... From 9cf0aaf654c51ce3d9e94891c381927803398c4f Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Fri, 14 Aug 2026 15:07:48 +0200 Subject: [PATCH 2/3] cleanup --- luxonis_eval/core/core.py | 11 +---------- luxonis_eval/core/runtime.py | 2 -- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/luxonis_eval/core/core.py b/luxonis_eval/core/core.py index dd93ddb..47a135f 100644 --- a/luxonis_eval/core/core.py +++ b/luxonis_eval/core/core.py @@ -190,16 +190,7 @@ def _run_evaluators(self) -> EvaluationResult: metric_update=metric_update_elapsed, ) - active_visualizer_cfgs = [ - visualizer_cfg - for visualizer_cfg in self.evaluator_cfg.visualizers - if visualizer_cfg.active - ] - for visualizer, _visualizer_cfg in zip( - self.visualizers, - active_visualizer_cfgs, - strict=True, - ): + for visualizer in self.visualizers: visualizer.visualize(predictions, self.engine.vis_frame()) progress.update(advance=1) diff --git a/luxonis_eval/core/runtime.py b/luxonis_eval/core/runtime.py index d26172f..706913b 100644 --- a/luxonis_eval/core/runtime.py +++ b/luxonis_eval/core/runtime.py @@ -1,6 +1,5 @@ import json from importlib.resources import files -from typing import Any import numpy as np from loguru import logger @@ -8,7 +7,6 @@ from luxonis_ml.data.utils import split_task from luxonis_ml.typing import Params -from luxonis_eval.config import EvaluatorConfig from luxonis_eval.core.context import EvalContext from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput From ef7a1fb22ffcbe1cab7558a27df2b11beae099f8 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Mon, 17 Aug 2026 14:03:18 +0200 Subject: [PATCH 3/3] README updates --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 229a4c5..2a16bb7 100644 --- a/README.md +++ b/README.md @@ -498,7 +498,9 @@ The framework consumes the returned `ModelSpec` to configure loader preprocessin Subclass [`BaseParser`](luxonis_eval/parsers/base_parser.py) and implement the single abstract method: -- **`parse(raw_output, **kwargs)`** - Convert raw engine output into a structured prediction format +- **`parse(output)`** - Convert raw engine output into a structured prediction format + +Parser configuration belongs in the parser itself, and LuxonisEval provides the remaining runtime information during setup. The parser bridges the gap between model-specific tensor layouts and the standardized prediction fields that downstream metrics expect. The built-in parsers populate [`Prediction`](luxonis_eval/parsers/predictions.py) as follows: @@ -515,7 +517,7 @@ Subclass [`BaseMetric`](luxonis_eval/metrics/base_metric.py) and implement the f - **`required_target_keys()`** - Declare which annotation keys the metric requires - **`reset()`** - Reset internal state such as counters or accumulators -- **`update(predictions, target, **kwargs)`** - Update the metric state for one sample +- **`update(predictions, target)`** - Update the metric state for one sample - **`compute()`** - Return the final metric values > [!IMPORTANT]