feat: support P2E bank DiffTest

api/services/presets.py

@@ -65,19 +65,11 @@ def create_code_agent_manager() -> ToolManager:
     return manager
 
 
-# Predefined tool set configurations
-PRESET_CONFIGS = {
-    "file_tools": {
-        "name": "File Operations",
-        "description": "Basic file system operations",
-        "tools": create_file_tools,
-    },
-    "code_agent": {
-        "name": "Code Agent",
-        "description": "Tools for code generation and manipulation",
-        "tools": create_code_agent_tools,
-    },
-}
+def preset_configs():
+    return {
+        "file_tools": create_file_tools,
+        "code_agent": create_code_agent_tools,
+    }
 
 
 def get_preset(name: str) -> List[Tool]:
@@ -93,14 +85,14 @@ def get_preset(name: str) -> List[Tool]:
     Raises:
         ValueError: If tool set does not exist
     """
-    config = PRESET_CONFIGS.get(name)
-    if not config:
-        available = ", ".join(PRESET_CONFIGS.keys())
+    presets = preset_configs()
+    factory = presets.get(name)
+    if factory is None:
+        available = ", ".join(presets)
         raise ValueError(f"Unknown preset: {name}. Available: {available}")
-
-    return config["tools"]()
+    return factory()
 
 
 def list_presets() -> List[str]:
     """List all available predefined tool sets"""
-    return list(PRESET_CONFIGS.keys())
+    return list(preset_configs())

api/steps/bebop/bemu/02_sim_event.step.py

@@ -27,10 +27,6 @@
 from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile_index, chip_emu_manifest
 
 
-PERFETTO_TARGETS = {
-    "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto",
-}
-
 config = {
     "name": "bebop-bemu-sim",
     "description": "Run bebop bemu emulator",
@@ -69,7 +65,12 @@ def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None:
     if Path(binary_name).name != binary_name:
         return None
 
-    return search_workload(workloads_output_root(bbdir), binary_name)
+    workload = search_workload(workloads_output_root(bbdir), binary_name)
+    if workload is not None:
+        return workload
+
+    kernel = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / binary_name
+    return str(kernel) if kernel.is_file() else None
 
 
 def is_native_host_elf(path: str) -> bool:
@@ -133,9 +134,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None:
         return
     ctx.logger.info(f"binary_path: {binary_path}")
     binary_dir = os.path.dirname(binary_path)
-    perfetto_target = (
-        PERFETTO_TARGETS.get(binary_name) if input_data.get("tool-profile") else None
-    )
+    perfetto_target = None
+    if input_data.get("tool-profile"):
+        if not binary_name.endswith("-run"):
+            raise ValueError(f"tool-profile binary must end with '-run': {binary_name}")
+        perfetto_target = f"{binary_name.removesuffix('-run')}-perfetto"
     if perfetto_target:
         clean_model_trace(binary_dir)
 

api/steps/bebop/bemu/03_batch_event.step.py

@@ -27,8 +27,7 @@
 from utils.stream_run import stream_run_logger_async
 from utils.event_common import check_result, get_origin_trace_id
 from bemu_common import bemu_manifest, chip_emu_manifest
-from regression import regression_workload_toml
-from regression_harness import nextest_harness_args
+from utils.workload_manifest import resolve_workload_toml
 
 config = {
     "name": "bebop-bemu-batch",
@@ -82,7 +81,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None:
     env.update(bebop_cargo_env(bbdir, chip))
     test_type = input_data.get("test", "elf-tests")
     try:
-        workload_toml = regression_workload_toml(
+        workload_toml = resolve_workload_toml(
             chip, "bemu", test_type, bbdir, rushB=rushB
         )
     except ValueError as e:
@@ -137,7 +136,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None:
         )
         ctx.logger.info("Cleaned previous bebop test artifacts")
 
-    harness = nextest_harness_args(workload_toml, elf_root, env)
+    harness = shlex.join([
+        "--",
+        "--workload-toml", workload_toml,
+        "--bb-tests-root", elf_root,
+        *(["--rushb-backend", "bemu"] if rushB else []),
+    ])
     nextest_cmd = (
         f"nix develop -c cargo nextest run --manifest-path {shlex.quote(str(bemu_cargo_manifest))} "
         "--test test_bemu "

api/steps/bebop/bemu/04_analysis_api.step.py

@@ -21,9 +21,6 @@
     "enqueues": ["bebop.bemu.analysis"],
 }
 
-ALLOWED = {"chip", "log-dir", "log_dir", "itrace", "mtrace"}
-
-
 def _fail(message: str, status: int = 400) -> ApiResponse:
     return ApiResponse(
         status=status,
@@ -38,7 +35,8 @@ def _fail(message: str, status: int = 400) -> ApiResponse:
 
 async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse:
     body = request.body or {}
-    unknown = sorted(k for k in body if k not in ALLOWED)
+    allowed = {"chip", "log-dir", "log_dir", "itrace", "mtrace"}
+    unknown = sorted(k for k in body if k not in allowed)
     if unknown:
         return _fail(f"unknown parameter(s): {', '.join(unknown)}")
 

api/steps/bebop/bemu/scripts/bemu_analysis.py

@@ -9,21 +9,8 @@
 from collections import Counter
 from pathlib import Path
 
-ISA = {
-    0: "fence",
-    1: "barrier",
-    16: "mvout",
-    32: "mset",
-    33: "mvin",
-    34: "mmio_set",
-    35: "mvin_mmio",
-}
-
-MATRIX_MNEMONICS = frozenset({"MATRIX", "MATRIX_F32", "SMATMUL_OS"})
-
-_API = Path(__file__).resolve().parents[4]
-if str(_API) not in sys.path:
-    sys.path.insert(0, str(_API))
+if str(Path(__file__).resolve().parents[4]) not in sys.path:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
 
 
 def abs_log_dir(value: object) -> Path:
@@ -42,6 +29,15 @@ def _hex_u(value: object, field: str, line_no: int) -> int:
 
 
 def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]:
+    system_isa = {
+        0: "fence",
+        1: "barrier",
+        16: "mvout",
+        32: "mset",
+        33: "mvin",
+        34: "mmio_set",
+        35: "mvin_mmio",
+    }
     path = (
         Path(bbdir)
         / "examples"
@@ -75,7 +71,7 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]:
     isa = ball["ballISA"]
     ball_path = ball["_file"]
 
-    names: dict[int, str] = dict(ISA)
+    names: dict[int, str] = dict(system_isa)
     matrix: set[int] = set()
     seen: set[int] = set()
     for entry in isa:
@@ -89,13 +85,13 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]:
             raise ValueError(f"ballISA mnemonic must be a non-empty string: {ball_path}")
         if funct in seen:
             raise ValueError(f"duplicate ballISA funct7 {funct}: {ball_path}")
-        if funct in ISA:
+        if funct in system_isa:
             raise ValueError(
-                f"ballISA funct7 {funct} ({mnemonic}) collides with ISA {ISA[funct]}: {ball_path}"
+                f"ballISA funct7 {funct} ({mnemonic}) collides with ISA {system_isa[funct]}: {ball_path}"
             )
         seen.add(funct)
         names[funct] = mnemonic.lower()
-        if mnemonic in MATRIX_MNEMONICS:
+        if mnemonic in {"MATRIX", "MATRIX_F32", "SMATMUL_OS"}:
             matrix.add(funct)
 
     mem = core["memdomain"]

api/steps/bebop/p2e/01_verilog_api.step.py

@@ -18,7 +18,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse:
         chip = require_chip(body)
     except ValueError as e:
         return ApiResponse(status=400, body={"error": str(e)})
-    data = {"chip": chip}
+    data = {"chip": chip, "diff": bool(body.get("diff", False))}
     if body.get("output_dir"):
         data["output_dir"] = body["output_dir"]
     await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}})

api/steps/bebop/p2e/02_buildbitstream_api.step.py

@@ -25,6 +25,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse:
 
     data = {
         "chip": chip,
+        "diff": bool(body.get("diff", False)),
         "vsrc_dir": vsrc_dir,
         "output_dir": body.get("output_dir") or body.get("output-dir"),
     }