Implement PPL evaluator for qwen 3 0.6B

scripts/e2e_eval/run_pytorch_baseline.py

@@ -152,16 +152,28 @@ def _measure_pytorch_latency(task_evaluator: Any, warmup: int, iterations: int)
 def _load_pytorch_model(model_id: str, task: str, device_str: str):
     """Load a native PyTorch model with the task-appropriate AutoModel class."""
     import torch
+
+    device = torch.device(
+        device_str if device_str != "cuda" or torch.cuda.is_available() else "cpu"
+    )
+
+    # text-generation resolves to a composite (embeddings/ctx/iter/lm_head) that
+    # is not a runnable HF model. Wrap a plain causal LM in the perplexity
+    # scoring contract so the evaluator scores it through the same path as a
+    # genai bundle.
+    if task == "text-generation":
+        from winml.modelkit.models.winml import HFCausalLM
+
+        _out(f"Loading causal LM baseline adapter for {model_id} on {device}")
+        return HFCausalLM(model_id, device)
+
     from transformers import AutoConfig
 
     from winml.modelkit.loader import resolve_task
 
     config = AutoConfig.from_pretrained(model_id)
     cls = resolve_task(config, task=task).model_class
     _out(f"Loading {cls.__name__} for {model_id} on {device_str}")
-    device = torch.device(
-        device_str if device_str != "cuda" or torch.cuda.is_available() else "cpu"
-    )
     return cls.from_pretrained(model_id).to(device).eval()
 
 
@@ -353,16 +365,25 @@ def main() -> None:
         metrics = task_evaluator.compute()
 
         if args.perf_iterations > 0:
-            latency = _measure_pytorch_latency(
-                task_evaluator,
-                warmup=args.perf_warmup,
-                iterations=args.perf_iterations,
-            )
-            _out(
-                f"PyTorch latency: mean={latency['mean_ms']}ms "
-                f"p50={latency['p50_ms']}ms p90={latency['p90_ms']}ms"
-            )
-            _emit_latency(latency)
+            if task == "text-generation":
+                # The perplexity evaluator's ``data`` items are token-ID lists
+                # and its ``pipe`` is None, so the HF-pipeline latency path does
+                # not apply. Genai baseline latency is out of scope here.
+                _out(
+                    "Skipping --perf-iterations: latency measurement is not "
+                    "supported for text-generation baselines."
+                )
+            else:
+                latency = _measure_pytorch_latency(
+                    task_evaluator,
+                    warmup=args.perf_warmup,
+                    iterations=args.perf_iterations,
+                )
+                _out(
+                    f"PyTorch latency: mean={latency['mean_ms']}ms "
+                    f"p50={latency['p50_ms']}ms p90={latency['p90_ms']}ms"
+                )
+                _emit_latency(latency)
 
         value = float(metrics[winml_metric_key])
         # Emit result as last stdout line (parsed by run_eval.py accuracy phase)

scripts/e2e_eval/utils/accuracy.py

@@ -25,6 +25,8 @@
     # WinML-vs-baseline delta is small — pick a tighter threshold than default.
     "knn_top1_accuracy": ("delta_relative", 0.02, 0.05, True),
     "pseudo_perplexity": ("delta_relative", 0.05, 0.10, False),
+    # Perplexity (text-generation PPL): lower is better.
+    "perplexity": ("delta_relative", 0.05, 0.10, False),
     # CER (OCR error rate): lower is better.
     "cer": ("delta_relative", 0.05, 0.10, False),
     # AbsRel (depth-estimation error): lower is better.

src/winml/modelkit/commands/eval.py

@@ -309,6 +309,7 @@ def eval(
     _resolve_reference(cfg)
     _apply_export_overrides(cfg, shape_config_path, input_specs, export_config, dynamic_axes)
     _resolve_device(cfg)
+    _resolve_genai_ep(ctx, cfg)
     _resolve_label_mapping(cfg)
     _run_dataset_script(cfg, trust_remote_code)
 
@@ -557,6 +558,36 @@ def _resolve_device(cfg: WinMLEvaluationConfig) -> None:
     console.print(f"[dim]Using device:[/dim] {resolved_target.device}")
 
 
+def _resolve_genai_ep(ctx: click.Context, cfg: WinMLEvaluationConfig) -> None:
+    """Turn an explicit ``--device`` into an EP override for a genai bundle.
+
+    A genai bundle mixes stages across EPs by design (its ``genai_config.json``
+    encodes the per-stage routing), so :class:`GenaiSession` only re-routes when
+    an EP override is present and otherwise leaves the bundle untouched. Passing
+    the device straight through would make ``--device`` a no-op: the whole point
+    of an explicit device is to force the pipeline onto it. Mirroring the
+    ``winml-genai`` perf precedence, an explicitly supplied ``--device`` is
+    resolved to a concrete EP (via the same device→EP path the ONNX runtime
+    uses), while the default (``auto``) respects the bundle's own routing.
+
+    A user-supplied ``--ep`` already forces the pipeline, so it wins untouched.
+    """
+    if cfg.ep is not None:
+        return
+    model_path = cfg.model_path
+    if not isinstance(model_path, str):
+        return
+    bundle = Path(model_path).expanduser()
+    if not (bundle.is_dir() and (bundle / "genai_config.json").is_file()):
+        return
+    if ctx.get_parameter_source("device") != click.core.ParameterSource.COMMANDLINE:
+        return
+
+    from ._perf_genai import resolve_genai_ep
+
+    cfg.ep = resolve_genai_ep(cfg.device)
+
+
 def _resolve_label_mapping(cfg: WinMLEvaluationConfig) -> None:
     """Load label-mapping JSON file (if any) into ``cfg.dataset.label_mapping``."""
     if cfg.dataset.label_mapping_file:
@@ -716,6 +747,16 @@ def _resolve_model_path(
                 "for preprocessor and config resolution."
             )
         return value, model_id
+
+    # An onnxruntime-genai bundle is a local *directory* (holding
+    # ``genai_config.json``), not a Hub model id. Route it to model_path so the
+    # genai loader reads the bundle from disk. Gate on the genai marker so a
+    # plain local HF checkpoint directory still flows through the model_id path
+    # (``from_pretrained``) as before.
+    expanded = Path(value).expanduser()
+    if expanded.is_dir() and (expanded / "genai_config.json").is_file():
+        return str(expanded), model_id
+
     if model_id is not None and model_id != value:
         raise click.UsageError(
             "Cannot pass both `-m <hf_id>` and `--model-id`. "

src/winml/modelkit/eval/__init__.py

@@ -39,6 +39,7 @@
     from .question_answering_evaluator import WinMLQuestionAnsweringEvaluator
     from .tensor_similarity_evaluator import TensorSimilarityEvaluator
     from .text_classification_evaluator import WinMLTextClassificationEvaluator
+    from .text_generation_evaluator import WinMLTextGenerationEvaluator
     from .token_classification_evaluator import WinMLTokenClassificationEvaluator
     from .zero_shot_classification_evaluator import WinMLZeroShotClassificationEvaluator
     from .zero_shot_image_classification_evaluator import WinMLZeroShotImageClassificationEvaluator
@@ -66,6 +67,8 @@
         ".question_answering_evaluator:WinMLQuestionAnsweringEvaluator",
     "WinMLTextClassificationEvaluator":
         ".text_classification_evaluator:WinMLTextClassificationEvaluator",
+    "WinMLTextGenerationEvaluator":
+        ".text_generation_evaluator:WinMLTextGenerationEvaluator",
     "WinMLTokenClassificationEvaluator":
         ".token_classification_evaluator:WinMLTokenClassificationEvaluator",
     "WinMLZeroShotClassificationEvaluator":
@@ -141,6 +144,7 @@ def __dir__() -> list[str]:
     "WinMLObjectDetectionEvaluator",
     "WinMLQuestionAnsweringEvaluator",
     "WinMLTextClassificationEvaluator",
+    "WinMLTextGenerationEvaluator",
     "WinMLTokenClassificationEvaluator",
     "WinMLZeroShotClassificationEvaluator",
     "WinMLZeroShotImageClassificationEvaluator",

src/winml/modelkit/eval/evaluate.py

@@ -23,6 +23,7 @@
 
     from ..models.winml.base import WinMLPreTrainedModel
     from ..models.winml.composite_model import WinMLCompositeModel
+    from ..models.winml.genai_causal_lm import WinMLGenaiCausalLM
     from .base_evaluator import WinMLEvaluator
 
 logger = logging.getLogger(__name__)
@@ -71,6 +72,8 @@
         "winml.modelkit.eval.tensor_similarity_evaluator:TensorSimilarityEvaluator",
     "mask-generation":
         "winml.modelkit.eval.mask_generation_evaluator:WinMLMaskGenerationEvaluator",
+    "text-generation":
+        "winml.modelkit.eval.text_generation_evaluator:WinMLTextGenerationEvaluator",
 }
 # fmt: on
 
@@ -201,6 +204,14 @@ def get_evaluator_class(config: WinMLEvaluationConfig) -> type[WinMLEvaluator]:
         "path": "mattmdjaga/human_parsing_dataset",
         "split": "train",
     },
+    "text-generation": {
+        # Raw wikitext-2 test split scored token-by-token for perplexity;
+        # ``input_column`` names the text field the corpus is built from.
+        "path": "Salesforce/wikitext",
+        "name": "wikitext-2-raw-v1",
+        "split": "test",
+        "columns_mapping": {"input_column": "text"},
+    },
 }
 
 
@@ -228,7 +239,9 @@ def to_dict(self) -> dict[str, Any]:
         return result
 
 
-def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCompositeModel | None:
+def _load_model(
+    config: WinMLEvaluationConfig,
+) -> WinMLPreTrainedModel | WinMLCompositeModel | WinMLGenaiCausalLM | None:
     """Load model from ONNX path or HF model ID.
 
     For evaluators that handle their own ORT session construction from a
@@ -243,11 +256,15 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo
     from ..session import EPDeviceTarget, WinMLEPRegistry, resolve_device
     from ..utils import cli as cli_utils
 
+    if config.task == "text-generation":
+        return _load_genai_causal_lm(config)
+
     # Two-ONNX compare: the evaluator builds both raw ORT sessions directly from
     # config.model_path / config.reference_path — no WinMLAutoModel / HF config.
     if config.mode == "compare" and config.reference_path is not None:
         return None
 
+
     quant_override: Any = None
     if not config.quant:
         from ..config import WinMLBuildConfig
@@ -344,6 +361,51 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo
         return _load_model(config)
 
 
+def _load_genai_causal_lm(config: WinMLEvaluationConfig) -> WinMLGenaiCausalLM:
+    """Load a causal LM from an onnxruntime-genai bundle directory.
+
+    ``-m <bundle_dir>`` resolves to ``config.model_path`` (a local directory),
+    so the bundle directory is read from there. ``ep`` / ``device`` pass straight
+    through: an explicit ``--ep`` (or an explicit ``--device`` resolved to an EP
+    by the CLI) forces the whole decoder pipeline onto that EP, while ``ep=None``
+    respects the bundle's ``genai_config.json`` routing. The session compiles the
+    bundle to EPContext when needed (reusing a cached ``_compiled/``), matching
+    the safety path ``winml perf`` relies on.
+
+    Raises:
+        ValueError: no bundle directory was provided, the path is not a
+            directory, or it is missing the ``genai_config.json`` / ONNX files
+            that mark a genai bundle.
+    """
+    from pathlib import Path
+
+    from ..models.winml.genai_causal_lm import WinMLGenaiCausalLM
+
+    bundle_path = config.model_path
+    if not bundle_path or isinstance(bundle_path, dict):
+        raise ValueError(
+            "text-generation evaluation requires a genai bundle *directory* via "
+            "-m <bundle_dir>."
+        )
+
+    bundle_dir = Path(bundle_path).expanduser()
+    if not bundle_dir.is_dir():
+        raise ValueError(f"Genai bundle directory not found: {bundle_dir}")
+    if not (bundle_dir / "genai_config.json").is_file():
+        raise ValueError(
+            f"'{bundle_dir}' is not a genai bundle: no genai_config.json found. "
+            "Point -m at a bundle built with 'winml build ... --device npu --ep qnn'."
+        )
+    if not any(bundle_dir.rglob("*.onnx")):
+        raise ValueError(f"'{bundle_dir}' contains no .onnx files; not a valid genai bundle.")
+
+    return WinMLGenaiCausalLM(
+        bundle_dir,
+        config.ep,
+        device=config.device,
+    )
+
+
 def _resolve_task(config: WinMLEvaluationConfig) -> str:
     """Resolve the eval task and validate it is supported.
 
@@ -405,11 +467,20 @@ def evaluate(config: WinMLEvaluationConfig) -> EvalResult:
             raise ValueError(
                 f"No dataset provided and no default for task '{config.task}'. Use --dataset."
             )
+        user_columns = dict(config.dataset.columns_mapping or {})
         for k, v in default.items():
             setattr(config.dataset, k, deepcopy(v))
+        # Preserve user-supplied --column values (e.g. the text-generation
+        # scoring parameters num_tokens / seqlen): the default mapping only
+        # fills columns the user did not provide, so an explicit --column wins.
+        config.dataset.columns_mapping = {
+            **deepcopy(default.get("columns_mapping", {})),
+            **user_columns,
+        }
         logger.warning(
             "--dataset not specified; attempting default dataset '%s' for task '%s'. "
-            "Any --split / --column / --streaming / --dataset-name options are ignored.",
+            "Any --split / --streaming / --dataset-name options are ignored; "
+            "explicit --column values are preserved.",
             config.dataset.path,
             config.task,
         )