Add two-ONNX comparison to winml eval --mode compare

docs/commands/eval.md

@@ -39,8 +39,9 @@ $ winml eval [options]
 | `--label-mapping` | | `PATH` | — | Path to a JSON file mapping dataset label names to the integer class IDs the model emits: `{"label_name": id}`. |
 | `--output` | `-o` | `PATH` | — | Output JSON file path for the evaluation results. |
 | `--schema` | | flag | `false` | Print the expected dataset schema for the given `--task` and exit. Does not run evaluation. |
-| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and the HuggingFace reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. |
+| `--mode` | | `onnx\|compare` | `onnx` | Evaluation mode. `onnx` evaluates the ONNX candidate on a dataset. `compare` runs the ONNX candidate and a reference on identical random inputs and reports per-tensor similarity metrics — no dataset required. The reference is the HuggingFace model from `--model-id` by default, or a second ONNX file when `--reference` is given. |
 | `--input-data` | | `PATH` | — | Path to a `.npz` file of real input tensors to compare with instead of randomly generated ones (used with `--mode compare`). Keys must match the candidate model's input names. The **leading axis of each array is the sample axis**, so an archive shaped `(N, ...)` yields `N` samples (mean/std/min/max are computed across them); all inputs must share the same `N`. Each run is shaped to the candidate's batch size — a dynamic batch runs one row per sample, a static batch `B` chunks the axis into `N // B` batches (trailing rows are dropped with a warning). Note this differs from `winml perf --input-data`, which runs the **whole archive as a single batch**. |
+| `--reference` | | `TEXT` | — | Reference `.onnx` file to compare the candidate against (used with `--mode compare`). Compares two ONNX models on identical random inputs; `--model-id` and `--task` are not required in this mode. Both models run on the same `--device` / `--ep`. |
 
 ## How it works
 
@@ -82,6 +83,12 @@ Compare an ONNX candidate against its HuggingFace reference on real input tensor
 $ winml eval --mode compare -m model.onnx --model-id microsoft/resnet-50 --input-data inputs.npz
 ```
 
+Compare two ONNX files directly (e.g. an fp32 baseline vs a quantized build), reporting per-output tensor-similarity metrics on identical random inputs — no `--model-id` or dataset needed:
+
+```bash
+$ winml eval --mode compare -m quantized.onnx --reference baseline.onnx
+```
+
 Check what dataset columns are expected before running, then remap them to match your dataset:
 
 ```bash

src/winml/modelkit/commands/eval.py

@@ -198,6 +198,17 @@
         "sample axis (N samples), and all inputs must share the same N."
     ),
 )
+@click.option(
+    "--reference",
+    "reference",
+    type=str,
+    default=None,
+    help=(
+        "Reference ONNX file to compare the candidate against (use with "
+        "--mode compare). Compares two ONNX models on identical random inputs; "
+        "--model-id / --task are not required in this mode."
+    ),
+)
 @cli_utils.skip_build_option()
 @cli_utils.format_option()
 @cli_utils.build_config_option()
@@ -240,6 +251,7 @@ def eval(
     show_schema: bool,
     mode: EvalMode,
     input_data: str | None,
+    reference: str | None,
     config_file: Path | None,
     skip_build: bool,
 ) -> None:
@@ -252,6 +264,8 @@ def eval(
 
         winml eval --mode compare -m cand.onnx --model-id microsoft/resnet-50 --input-data data.npz
 
+        winml eval --mode compare -m cand.onnx --reference baseline.onnx
+
     Run `winml eval --schema --task <task>` to see the dataset columns
     and options expected by each task.
     """
@@ -287,8 +301,12 @@ def eval(
     if cfg.input_data is not None and cfg.mode != "compare":
         raise click.UsageError("--input-data is only valid with --mode compare.")
 
+    if cfg.reference_path is not None and cfg.mode != "compare":
+        raise click.UsageError("--reference is only valid with --mode compare.")
+
     # ── 2. Resolve in place ──
-    _resolve_model(cfg, model, model_id)
+    _resolve_model(cfg, model, model_id, allow_missing_model_id=cfg.reference_path is not None)
+    _resolve_reference(cfg)
     _apply_export_overrides(cfg, shape_config_path, input_specs, export_config, dynamic_axes)
     _resolve_device(cfg)
     _resolve_label_mapping(cfg)
@@ -420,13 +438,54 @@ def _resolve_model(
     cfg: WinMLEvaluationConfig,
     model: tuple[str, ...],
     model_id: str | None,
+    *,
+    allow_missing_model_id: bool = False,
 ) -> None:
     """Resolve ``-m`` / ``--model-id`` into ``cfg.model_path`` / ``cfg.model_id``."""
-    model_path, resolved_id = _resolve_model_path(model=model, model_id=model_id)
+    model_path, resolved_id = _resolve_model_path(
+        model=model, model_id=model_id, allow_missing_model_id=allow_missing_model_id
+    )
     cfg.model_path = model_path
     cfg.model_id = resolved_id
 
 
+def _resolve_reference(cfg: WinMLEvaluationConfig) -> None:
+    """Validate and normalize ``cfg.reference_path`` for two-ONNX compare.
+
+    Requires the candidate (``-m``) to be a single ONNX file (composite
+    ``role=path`` candidates and build-from-id are not supported with
+    ``--reference`` yet). Resolves Hub-hosted ONNX refs to local paths.
+    """
+    if cfg.reference_path is None:
+        return
+
+    if not isinstance(cfg.model_path, str):
+        raise click.UsageError(
+            "--reference requires the candidate (-m) to be a single ONNX file. "
+            "Composite (role=path) candidates and build-from-id are not "
+            "supported with --reference."
+        )
+
+    ref = cfg.reference_path
+    if Path(ref).suffix.lower() != ".onnx":
+        raise click.BadParameter(
+            f"--reference must be an .onnx file, got: {ref}",
+            param_hint="--reference",
+        )
+    try:
+        ref = cli_utils.normalize_model_arg(ref) or ref
+    except Exception as e:
+        raise click.ClickException(
+            f"Failed to resolve Hub-hosted reference ONNX path {ref!r}: {e}"
+        ) from e
+    if not Path(ref).exists():
+        raise click.BadParameter(
+            f"Reference ONNX file not found: {ref}",
+            param_hint="--reference",
+        )
+    cfg.reference_path = ref
+
+
 def _apply_export_overrides(
     cfg: WinMLEvaluationConfig,
     shape_config_path: Path | None,
@@ -567,8 +626,14 @@ def _resolve_model_path(
     *,
     model: tuple[str, ...],
     model_id: str | None,
+    allow_missing_model_id: bool = False,
 ) -> tuple[str | dict[str, str] | None, str | None]:
-    """Turn repeated -m values + --model-id into (model_path, model_id)."""
+    """Turn repeated -m values + --model-id into (model_path, model_id).
+
+    When ``allow_missing_model_id`` is set (two-ONNX ``--mode compare``), a
+    plain ``-m <file>.onnx`` is accepted without ``--model-id`` because the
+    candidate runs as a raw ORT session with no HF config resolution.
+    """
     if not model:
         if model_id is not None:
             return None, model_id
@@ -644,6 +709,8 @@ def _resolve_model_path(
                 param_hint="-m/--model",
             )
         if model_id is None:
+            if allow_missing_model_id:
+                return value, None
             raise click.UsageError(
                 "When using an ONNX file, --model-id is required "
                 "for preprocessor and config resolution."
@@ -682,11 +749,20 @@ def display_eval_report(result: EvalResult, console: Console) -> None:
     # archive (via EvalResult.num_samples), not the unused config default.
     samples = result.num_samples if result.num_samples is not None else ds.samples
 
-    # Header
+    # Header — model_id when building from HF, otherwise the ONNX path(s). A
+    # composite model_path is a {role: path} dict; join its paths so the title
+    # stays a readable string instead of a raw dict repr.
+    if cfg.model_id:
+        eval_name = cfg.model_id
+    elif isinstance(cfg.model_path, dict):
+        eval_name = ", ".join(str(path) for path in cfg.model_path.values())
+    else:
+        eval_name = str(cfg.model_path)
+
     console.print()
     console.print(
         Panel.fit(
-            f"[bold]Evaluation: {cfg.model_id}[/bold]",
+            f"[bold]Evaluation: {eval_name}[/bold]",
             border_style="blue",
         )
     )
@@ -700,8 +776,13 @@ def display_eval_report(result: EvalResult, console: Console) -> None:
     elif ds.path:
         console.print(f"[dim]Dataset:[/dim]    {ds.path}")
     console.print(f"[dim]Samples:[/dim]    {samples}")
-    if cfg.model_path:
+    if isinstance(cfg.model_path, dict):
+        for role, path in cfg.model_path.items():
+            console.print(f"[dim]ONNX ({role}):[/dim] {path}")
+    elif cfg.model_path:
         console.print(f"[dim]ONNX:[/dim]       {cfg.model_path}")
+    if cfg.reference_path:
+        console.print(f"[dim]Reference:[/dim]  {cfg.reference_path}")
 
     # Metrics table
     console.print()

src/winml/modelkit/eval/config.py

@@ -113,6 +113,10 @@ class WinMLEvaluationConfig:
             randomly generated ones. The leading axis of each array is the sample
             axis, so one archive can hold ``N`` samples; all inputs must share the
             same leading length.
+        reference_path: Path to a second ``.onnx`` file used as the reference in
+            ``--mode compare``. When set, both ``model_path`` and ``reference_path``
+            run as raw ORT sessions and their output tensors are compared directly,
+            so no ``model_id`` / ``task`` / HF reference is needed.
         task: HF pipeline task. Auto-detected from model_id if omitted.
         device: Target device for inference.
         ep: Explicit execution provider (e.g., "qnn", "dml"). Overrides
@@ -132,7 +136,8 @@ class WinMLEvaluationConfig:
               labeled dataset.
             - ``"compare"``: compare ONNX vs HF reference output tensors
               on identical random inputs and report tensor-similarity
-              metrics per output tensor.
+              metrics per output tensor. When ``reference_path`` is set,
+              the reference is a second ONNX file instead of the HF model.
 
     Usage:
         config = WinMLEvaluationConfig(
@@ -144,6 +149,7 @@ class WinMLEvaluationConfig:
     model_id: str | None = None
     model_path: str | dict[str, str] | None = None
     input_data: str | None = None
+    reference_path: str | None = field(default=None, metadata={"cli_name": "reference"})
     task: str | None = None
     device: str = "auto"
     precision: str = "auto"
@@ -177,6 +183,8 @@ def to_dict(self) -> dict:
             result["model_path"] = self.model_path
         if self.input_data is not None:
             result["input_data"] = self.input_data
+        if self.reference_path is not None:
+            result["reference_path"] = self.reference_path
         if self.task is not None:
             result["task"] = self.task
         result["device"] = self.device
@@ -229,6 +237,7 @@ def from_dict(cls, data: dict) -> WinMLEvaluationConfig:
             model_id=data.get("model_id"),
             model_path=data.get("model_path"),
             input_data=data.get("input_data"),
+            reference_path=data.get("reference_path"),
             task=data.get("task"),
             device=data.get("device", "auto"),
             precision=data.get("precision", "auto"),

src/winml/modelkit/eval/evaluate.py

@@ -243,6 +243,11 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo
     from ..session import EPDeviceTarget, WinMLEPRegistry, resolve_device
     from ..utils import cli as cli_utils
 
+    # Two-ONNX compare: the evaluator builds both raw ORT sessions directly from
+    # config.model_path / config.reference_path — no WinMLAutoModel / HF config.
+    if config.mode == "compare" and config.reference_path is not None:
+        return None
+
     quant_override: Any = None
     if not config.quant:
         from ..config import WinMLBuildConfig
@@ -385,8 +390,14 @@ def evaluate(config: WinMLEvaluationConfig) -> EvalResult:
     mode = config.mode if config.mode is not None else "onnx"
     if mode not in EVAL_MODES:
         raise ValueError(f"Invalid mode {mode!r}; expected one of {EVAL_MODES} or None.")
+    # Two-ONNX compare: both candidate and reference run as raw ORT sessions, so
+    # HF task resolution / model_id are not required — keep task as-is.
+    onnx_compare = mode == "compare" and config.reference_path is not None
     config = replace(
-        config, mode=mode, task=_resolve_task(config), dataset=deepcopy(config.dataset)
+        config,
+        mode=mode,
+        task=config.task if onnx_compare else _resolve_task(config),
+        dataset=deepcopy(config.dataset),
     )
     if config.mode != "compare" and config.dataset.path is None:
         default = _DEFAULT_DATASETS.get(config.task) if config.task is not None else None
@@ -457,12 +468,16 @@ def print_config(config: WinMLEvaluationConfig) -> None:
     """Print effective evaluation config to the console (quantize.py style)."""
     ds = config.dataset
     output_console = Console()
-    output_console.print(f"[bold blue]Model:[/bold blue] {config.model_id}")
+    if config.model_id is not None:
+        output_console.print(f"[bold blue]Model:[/bold blue] {config.model_id}")
     if config.model_path is not None:
         output_console.print(f"[bold blue]Model path:[/bold blue] {config.model_path}")
     if config.input_data is not None:
         output_console.print(f"[bold blue]Input data:[/bold blue] {config.input_data}")
-    output_console.print(f"[bold blue]Task:[/bold blue] {config.task}")
+    if config.reference_path is not None:
+        output_console.print(f"[bold blue]Reference:[/bold blue] {config.reference_path}")
+    if config.task is not None:
+        output_console.print(f"[bold blue]Task:[/bold blue] {config.task}")
     output_console.print(f"[bold blue]Device:[/bold blue] {config.device}")
     if config.ep is not None:
         output_console.print(f"[bold blue]EP:[/bold blue] {config.ep}")