added terminal bench 2 benchmarks support

.gitignore

@@ -10,4 +10,6 @@ dist
 history.json
 tools*.json
 
-README_TECHNICAL.md
+README_TECHNICAL.md
+
+jobs/

README.md

@@ -39,6 +39,14 @@ mini-code --enable-shell --auto-mode --agent-md "./skill.md" --allowed-dir "./"
 Write an agent.md file for me that helps an LLM write efficient triton kernels, given a reference implementation.
 ```
 
+Run a single prompt and exit:
+```bash
+mini-code --prompt "Explain this codebase" --exit-after-run
+```
+
+Combine `--exit-after-run` with `--auto-mode` to keep executing tool calls until
+the tool loop finishes, then exit.
+
 4. Get help:
 ```bash
 mini-code --help
@@ -62,11 +70,12 @@ options:
   --auto-mode           Whether to run the agent in `auto-mode'. Or default: `manual-mode'.
   --agent-md AGENT_MD   If the agent should use an agent-md file... (it will added after system message.)
   --prompt PROMPT       An initial prompt from the user...
+  --exit-after-run      Exit after the initial prompt finishes. With --auto-mode, wait until the tool loop finishes.
   --ask-permission      Ask for permission before any tool call.
   --allowed-dir ALLOWED_DIR
                         Allowed directory for file operations
   --enable-shell        Allow shell execution. Default: False
   ```
 
 
-## (C) Nikolai Rozanov, 2026 - Present
+## (C) Nikolai Rozanov, 2026 - Present

benchmarks/README.md

@@ -0,0 +1,88 @@
+# Running Mini Code with Harbor
+
+[Harbor](https://harborframework.com/) runs Mini Code in benchmark task
+containers and records the results. The adapter uses your current working tree,
+not the published package.
+
+## Setup
+
+Install Docker (and make sure Docker desktop is running) and Ollama. Then run this once from the repository root:
+
+```shell
+uv tool install --force --with-editable . harbor
+ollama pull qwen2.5-coder:7b
+```
+
+This installs [Harbor](https://harborframework.com/docs/getting-started) with
+the local benchmark adapter.
+
+## Local smoke test
+
+Run one Terminal-Bench 2 task on macOS:
+
+```shell
+harbor run \
+  --dataset terminal-bench@2.0 \
+  --include-task-name break-filter-js-from-html \
+  --agent benchmarks.harbor_agent:MiniCodeAgent \
+  --model qwen2.5-coder:7b \
+  --ak base_url=http://host.docker.internal:11434 \
+  --allow-agent-host host.docker.internal \
+  -n 1
+```
+
+Add `--install-only` to test container setup without running the model.
+
+## Full Terminal-Bench 2 run
+
+Run the full Terminal-Bench 2 dataset with four concurrent tasks and three
+attempts per task:
+
+```shell
+harbor run \
+  --dataset terminal-bench@2.0 \
+  --agent benchmarks.harbor_agent:MiniCodeAgent \
+  --model qwen2.5-coder:7b \
+  --ak base_url=http://host.docker.internal:11434 \
+  --allow-agent-host host.docker.internal \
+  -n 4 \
+  -k 3
+```
+
+## Remote endpoint
+
+Pass the API key through the agent environment:
+
+```shell
+harbor run \
+  --dataset terminal-bench@2.0 \
+  --include-task-name break-filter-js-from-html \
+  --agent benchmarks.harbor_agent:MiniCodeAgent \
+  --model provider/model-name \
+  --ak base_url=https://api.example.com \
+  --ae MINI_CODE_API_KEY="$MINI_CODE_API_KEY" \
+  --allow-agent-host api.example.com \
+  -n 1
+```
+
+Mini Code appends `/v1/chat/completions` to `base_url`.
+
+## Results
+
+Harbor saves runs under `jobs/<timestamp>/`. Useful files include:
+
+- `result.json`: rewards and run summary
+- `agent/mini-code.log`: Mini Code console output
+- `agent/history_<session-id>.json`: conversation history
+
+
+
+## Troubleshooting
+
+- `No module named 'benchmarks'`: rerun the editable Harbor installation above.
+- Model unreachable: check the endpoint and its `--allow-agent-host` entry.
+- Zero reward: inspect `mini-code.log`, `trial.log`, verifier output, and
+`result.json`.
+
+To use another Harbor dataset, replace `--dataset terminal-bench@2.0` and
+adjust or remove `--include-task-name`.

benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark integrations for Mini Code CLI."""

benchmarks/harbor_agent.py

@@ -0,0 +1,159 @@
+"""Harbor installed-agent adapter for the current Mini Code CLI checkout."""
+
+from __future__ import annotations
+
+import re
+import shlex
+import tarfile
+import tempfile
+from pathlib import Path
+from typing import override
+
+from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+
+_REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+_ARCHIVE_PATH = "/installed-agent/mini_code_cli.tar.gz"
+_UV_ENV = {
+    "UV_TOOL_BIN_DIR": "/usr/local/bin",
+    "UV_TOOL_DIR": "/opt/uv-tools",
+}
+
+
+def create_source_archive(repository_root: Path, destination: Path) -> Path:
+    """Create an installable source archive containing only project inputs."""
+    repository_root = repository_root.resolve()
+    required_files = [
+        repository_root / "pyproject.toml",
+        repository_root / "README.md",
+        repository_root / "LICENSE",
+    ]
+    missing = [path.name for path in required_files if not path.is_file()]
+    package_root = repository_root / "mini_code_cli"
+    if not package_root.is_dir():
+        missing.append("mini_code_cli/")
+    if missing:
+        raise FileNotFoundError(
+            f"Cannot build Mini Code source archive; missing: {', '.join(missing)}"
+        )
+
+    version_match = re.search(
+        r'(?m)^version\s*=\s*"([^"]+)"\s*$',
+        (repository_root / "pyproject.toml").read_text(encoding="utf-8"),
+    )
+    if version_match is None:
+        raise ValueError("Could not determine project version from pyproject.toml")
+    archive_root = f"mini_code_cli-{version_match.group(1)}"
+
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(destination, mode="w:gz") as archive:
+        for path in required_files:
+            archive.add(path, arcname=f"{archive_root}/{path.name}")
+        for path in sorted(package_root.rglob("*.py")):
+            relative_path = path.relative_to(repository_root)
+            archive.add(path, arcname=f"{archive_root}/{relative_path}")
+    return destination
+
+
+class MiniCodeAgent(BaseInstalledAgent):
+    """Install and run the checked-out Mini Code CLI inside a Harbor task."""
+
+    def __init__(self, *args, base_url: str | None = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.base_url = base_url
+
+    @staticmethod
+    @override
+    def name() -> str:
+        return "mini-code"
+
+    @override
+    def get_version_command(self) -> str | None:
+        return (
+            "UV_TOOL_BIN_DIR=/usr/local/bin UV_TOOL_DIR=/opt/uv-tools "
+            "uv tool list"
+        )
+
+    @override
+    def parse_version(self, stdout: str) -> str:
+        match = re.search(r"(?m)^mini-code-cli\s+v?(\S+)", stdout)
+        return match.group(1) if match else stdout.strip()
+
+    @override
+    async def install(self, environment: BaseEnvironment) -> None:
+        with tempfile.TemporaryDirectory(prefix="mini-code-harbor-") as temp_dir:
+            archive_path = create_source_archive(
+                _REPOSITORY_ROOT,
+                Path(temp_dir) / "mini_code_cli.tar.gz",
+            )
+            await environment.upload_file(archive_path, _ARCHIVE_PATH)
+
+        bootstrap_command = (
+            "set -euo pipefail; "
+            "if ! command -v curl >/dev/null 2>&1; then "
+            "if command -v apt-get >/dev/null 2>&1; then "
+            "apt-get update && "
+            "DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates; "
+            "elif command -v apk >/dev/null 2>&1; then "
+            "apk add --no-cache curl ca-certificates; "
+            "elif command -v dnf >/dev/null 2>&1; then "
+            "dnf install -y curl ca-certificates; "
+            "elif command -v yum >/dev/null 2>&1; then "
+            "yum install -y curl ca-certificates; "
+            "else echo 'No supported package manager found to install curl' >&2; "
+            "exit 1; fi; fi; "
+            "if ! command -v uv >/dev/null 2>&1; then "
+            "curl -LsSf https://astral.sh/uv/install.sh "
+            "| env UV_INSTALL_DIR=/usr/local/bin sh; "
+            "fi; "
+            f"uv tool install --force {_ARCHIVE_PATH}; "
+            "uv tool list"
+        )
+        await self.exec_as_root(
+            environment,
+            command=bootstrap_command,
+            env={**_UV_ENV, "PYTHONUNBUFFERED": "1"},
+        )
+
+    def _build_run_command(self, instruction: str) -> str:
+        if not self.model_name:
+            raise ValueError("Harbor must provide a model with --model")
+        if not self.base_url:
+            raise ValueError(
+                "Harbor must provide an OpenAI-compatible endpoint with "
+                "--ak base_url=<url>"
+            )
+
+        return shlex.join(
+            [
+                "mini-code",
+                "--auto-mode",
+                "--enable-shell",
+                "--exit-after-run",
+                "--cache-dir",
+                "/logs/agent",
+                "--model",
+                self.model_name,
+                "--url",
+                self.base_url,
+                "--prompt",
+                instruction,
+            ]
+        )
+
+    @override
+    @with_prompt_template
+    async def run(
+        self,
+        instruction: str,
+        environment: BaseEnvironment,
+        context: AgentContext,
+    ) -> None:
+        await self.exec_as_agent(
+            environment,
+            command=self._build_run_command(instruction),
+            env={"PYTHONUNBUFFERED": "1"},
+            cwd=environment.task_env_config.workdir,
+        )

mini_code_cli/agent_loop.py

@@ -54,7 +54,12 @@ def parse_args():
     # MODEL / LLM related
     parser.add_argument("--url", type=str, default="http://0.0.0.0:30000", help="API host")
     parser.add_argument("--max-tokens", type=int, help="Maximum tokens for LLM response")
-    parser.add_argument("--temperature", type=float, default=0.0, help="Temperature for LLM")
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=None,
+        help="Temperature for LLM. If omitted, use the model provider's default.",
+    )
     parser.add_argument("--model", type=str, default="default", help="Model name")
     parser.add_argument("--api-key", type=str, default=None, help="API key for authentication. If not set, it tries to find it in env variable: MINI_CODE_API_KEY.")
 
@@ -66,6 +71,11 @@ def parse_args():
     parser.add_argument("--auto-mode", action="store_true", help="Whether to run the agent in `auto-mode'. Or default: `manual-mode'.")
     parser.add_argument("--agent-md", type=str, help="If the agent should use an agent-md file... (it will added after system message.)")
     parser.add_argument("--prompt", type=str, help="An initial prompt from the user...")
+    parser.add_argument(
+        "--exit-after-run",
+        action="store_true",
+        help="Exit after the initial prompt finishes. With --auto-mode, wait until the tool loop finishes.",
+    )
 
     # Agent permissions related
     parser.add_argument("--ask-permission", action="store_true", help="Ask for permission before any tool call.")
@@ -346,6 +356,9 @@ def main():
 
             save_history(messages, args, unique_id)
 
+            if args.exit_after_run and (not args.auto_mode or not llm_tool_response_flag):
+                break
+
         except KeyboardInterrupt:
             print(f"{BOLD}{RED}INTERRUPTING:{RESET} Mini Code Exiting...")
 
@@ -357,4 +370,4 @@ def main():
             save_history(messages, args, unique_id)
 
 if __name__ == "__main__":
-    main()
+    main()

mini_code_cli/utils.py

@@ -3,7 +3,7 @@
 import argparse
 
 
-def call_openai_server(messages, max_tokens=2048, temperature=0.0, top_p=0.95, model="default", server_url=None, api_key=None, tools=None,):
+def call_openai_server(messages, max_tokens=2048, temperature=None, top_p=0.95, model="default", server_url=None, api_key=None, tools=None,):
     """
     Querying an OpenAI-compatible server (local or remote) using the requests library.
     Supports authentication via API key if provided.
@@ -19,11 +19,13 @@ def call_openai_server(messages, max_tokens=2048, temperature=0.0, top_p=0.95, m
     payload = {
         "model": model,
         "messages": messages,
-        "temperature": temperature,
         # "max_tokens": max_tokens,
         # "top_p": top_p
     }
 
+    if temperature is not None:
+        payload["temperature"] = temperature
+
     if tools:
         payload["tools"] = tools
         payload["tool_choice"] = "auto"
@@ -44,14 +46,24 @@ def call_openai_server(messages, max_tokens=2048, temperature=0.0, top_p=0.95, m
         return content, tool_calls
 
     except requests.exceptions.RequestException as e:
+        error_detail = ""
+        if e.response is not None:
+            try:
+                error_payload = e.response.json()
+                error_detail = error_payload.get("error", {}).get("message", "")
+            except (ValueError, AttributeError):
+                error_detail = e.response.text.strip()
+
         print(f"Error calling OpenAI Server...: {e}")
+        if error_detail:
+            print(f"API error: {error_detail}")
         return None, None
     except (json.JSONDecodeError, KeyError, IndexError) as e:
         print(f"Error processing response: {e}")
         return None, None
 
 # Alternative: use the newer SGLang API format (v0.3+)
-def call_openai_server_prompt(prompt, max_tokens=1024, temperature=0.0, top_p=0.95, model="default", server_url=None, api_key=None, tools=None):
+def call_openai_server_prompt(prompt, max_tokens=1024, temperature=None, top_p=0.95, model="default", server_url=None, api_key=None, tools=None):
     """
     OpenAI compatible call, pre-creating the messages object.
     """