diff --git a/.github/workflows/benchmark-onboarding.yml b/.github/workflows/benchmark-onboarding.yml new file mode 100644 index 000000000..0178bd4c4 --- /dev/null +++ b/.github/workflows/benchmark-onboarding.yml @@ -0,0 +1,28 @@ +name: Benchmark onboarding +on: + pull_request: + paths: + - 'agents/**' + - 'benchmarks/**' + - 'examples/agents/**' + - 'tests/benchmark/**' + - 'data_v0.1_multi/v1.3*/**' + - 'analysis/scripts/compute_cce_v2.py' + - 'analysis/requirements-cce.txt' + - '.github/workflows/benchmark-onboarding.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + offline-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r agents/requirements.txt -r analysis/requirements-cce.txt + - run: python -m unittest discover -s tests/benchmark -v + - run: python agents/run.py --help + - run: python benchmarks/score.py --help + - run: python analysis/scripts/compute_cce_v2.py --help diff --git a/.gitignore b/.gitignore index ed01fd42b..a5f5f8605 100644 --- a/.gitignore +++ b/.gitignore @@ -466,3 +466,10 @@ benchmark_results **/results_*/ scripts/run_parallel_45_60.sh scripts/run_tasks_45_60.sh + +# Local benchmark configuration and generated artifacts +.env +.env.local +.venv/ +/runs/ +/agents/configs/*.local.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..39ba1b376 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Contributing to AgentWorld + +Start with the [repository map](docs/repository-map.md) and +[benchmark quickstart](docs/benchmark/quickstart.md). + +- **Model or harness integrations:** supply a minimal adapter/configuration, + a smoke-task trajectory, and the exact reproduction command. +- **Task/verifier fixes:** include a failing example and a regression check. + Describe which scores or historical runs may be affected; do not silently + change benchmark criteria as part of a formatting cleanup. +- **Documentation:** use root-relative commands and environment-based examples. + Separate portable workflows from machine-specific experiment launchers. +- **Result reviews:** use the [GitHub issue submission workflow](docs/benchmark/scoring.md). + Check configuration, coverage, protocol, and raw evidence before publication. + +Run the offline onboarding checks without game credentials or model calls: + +```bash +python -m pip install -r agents/requirements.txt +python -m unittest discover -s tests/benchmark -v +python agents/run.py --help +python benchmarks/score.py --help +``` + +Live game/model runs are separate validation. Include the task/harness revision, +settings, and coverage with those results. Keep credentials, virtual environments, +and generated run artifacts out of commits. diff --git a/README.md b/README.md index b488ce654..45afce2ce 100755 --- a/README.md +++ b/README.md @@ -1,417 +1,46 @@ # AgentWorld -![MPL-2.0 License](https://img.shields.io/github/license/Kaetram/Kaetram-Open) [![Website](https://img.shields.io/website?url=https%3A%2F%2Fagentworld.io&style=flat)](https://agentworld.io) - -AgentWorld is a collaborative gaming environment where AI agents can interact, explore, and play together in a 2D multiplayer world. Built as a research platform for multi-agent AI systems, AgentWorld enables developers to create, control, and study AI agents in a rich gaming environment. - -Key features for AI research and development: -- **Multi-agent coordination**: Agents can collaborate, compete, and communicate with each other -- **Rich observation space**: Comprehensive API for agents to perceive their environment -- **Action space**: Agents can move, chat, fight, craft, and interact with the world -- **Persistent world**: Agents can learn and adapt over time in a persistent environment -- **Research-friendly**: Built-in APIs for easy integration with AI frameworks - -## Credits - -AgentWorld is built upon the excellent open-source foundation of [Kaetram](https://github.com/Kaetram/Kaetram-Open/tree/master), specifically based on the Kaetram master branch. Kaetram is an open-source RPG game that expands on Little Workshop's BrowserQuest. We're grateful to the Kaetram community for creating such a robust and well-designed game engine that serves as the perfect foundation for multi-agent AI research. - - -## Get Started - -### Prerequisites - -You must first [install Node.js](https://nodejs.org/en/download) to run the project, and -_optionally_ [install MongoDB](https://www.mongodb.com/try/download/community) to store user data on -the server. - -#### NOTE: Node.js - -> We recommend using Node.js version `v20` or higher, following the -> [Long Term Support (LTS) schedule](https://nodejs.org/en/about/releases), to have the most stable -> experience when developing/experimenting with AgentWorld. Older versions may not work with our -> current dependencies and package manager. - -#### NOTE: MongoDB - -> MongoDB is not a requirement for AgentWorld to run, but you can store and save user data if you -> install it and run an online environment with all the features enabled. To do this, see -> [Configuration](#configuration), and set `SKIP_DATABASE=false`. _If you do choose to install -> MongoDB, a user is not necessary, but you can enable authentication with the `MONGODB_AUTH` -> setting._ - -#### Yarn - -You will also need to enable [Yarn](https://yarnpkg.com) through [Corepack](https://nodejs.org/dist/latest/docs/api/corepack.html), to manage the dependencies. - -> The preferred way to manage Yarn is through [Corepack](https://nodejs.org/dist/latest/docs/api/corepack.html), a new binary shipped with all Node.js releases [...] -> -> To enable it, run the following command: -> -> ```console -> corepack enable -> ``` -> -> - -### Installing - -Install the dependencies by simply running - -```console -yarn -``` - -### Configuration - -Copy and rename [`.env.defaults`](.env.defaults) to `.env`, and modify the contents to fit your -needs. - -```console -cp .env.defaults .env -``` - -Enable Mongodb by setting `SKIP_DATABASE=false` in the `.env` file. - - -### Running - - -To run live development builds, use - -```console -yarn dev -``` - -To create production builds, run - -```console -yarn build -``` - -Then, to run each production build, use - -```console -yarn start -``` - -We recommend to run first with `yarn dev` to start the server in development mode, so that you can see the changes you make to the code in real time. - - -## Run Agent Console - - -AgentWorld comes with a CLI tool to deploy an agent and interact with it using natural language. Make sure the game server is running, then in another terminal run: - -```console -python agents/console.py \ ---provider openai \ ---model gpt-4o \ ---username test1 \ ---password test1 -``` - -### Advanced Agent Configuration - -You can configure the agent's initial state using command line options: - -```console -# Set initial location and combat skills -python agents/console.py \ ---provider openai \ ---model gpt-4o \ ---username test1 \ ---password test1 \ ---location 300,200 \ ---combat-level-accuracy 45 \ ---combat-level-strength 50 \ ---combat-level-magic 30 - -# Start with equipped items and inventory -python agents/console.py \ ---provider openai \ ---model gpt-4o \ ---username test1 \ ---password test1 \ ---equipped-items bastardsword whitearmor ironhelmet:1:3 \ ---inventory-items healingpotion:10 firepotion:5 stick:20 - -# Full configuration example -python agents/console.py \ ---provider openai \ ---model gpt-4o \ ---username test1 \ ---password test1 \ ---location 250,180 \ ---combat-level-accuracy 60 \ ---combat-level-strength 60 \ ---combat-level-defense 50 \ ---combat-level-health 70 \ ---combat-level-magic 40 \ ---combat-level-archery 35 \ ---equipped-items bastardsword:1:2 whitearmor goldboots \ ---inventory-items healingpotion:20 firepotion:10 pythararrow:100 \ ---task "explore the world and fight strong enemies" -``` - -#### Initial State Options - -- `--location x,y`: Teleport to coordinates at startup (e.g., `--location 250,180`) -- `--combat-level-xxx N`: Set specific combat skill levels (1-120) - - Available skills: accuracy, strength, defense, health, magic, archery - - HP and MP are automatically restored to maximum after setting combat levels -- `--equipped-items item1 item2:count:enchant`: Set initial equipped items -- `--inventory-items item1:count item2:count:enchant`: Set initial inventory items - -#### Item Format - -- Basic: `itemkey` (e.g., `coppersword`) -- With count: `itemkey:count` (e.g., `healingpotion:10`) -- With enchantment: `itemkey:count:enchant` (e.g., `ironhelmet:1:3`) - -This will launch a console where you can interact with the agent. - -#### Cheat Commands - -You can use cheat commands both interactively and with the `--task` option: - -- `/teleport x y` or `/teleport spawn` - Teleport to coordinates -- `/equip itemkey [count] [enchant]` - Give and equip item -- `/setlevel level` - Set all combat skills to level -- `/give itemkey [count]` - Give items to inventory -- `/fullequip` - Give essential equipment set -- `/observe [radius]` - Show raw observation JSON data - -Examples: -```bash -# Interactive mode -๐ŸŽฎ What would you like to do? > /observe 32 - -# Single-task mode -python agents/console.py --task "/observe 32" --username test1 --password test1 -python agents/console.py --task "/teleport 300 200" --username test1 --password test1 -python agents/console.py --task "/setlevel 50" --username test1 --password test1 - -# Create fresh characters (no password needed) -python agents/console.py --new-character --username freshbot --task "/observe" -python agents/console.py --new-character --username testchar --location 300,200 --combat-level-strength 45 -``` - -#### New Character Creation - -Use `--new-character` to create fresh characters or reset existing ones: - -- **No password required** - The system handles authentication automatically -- **Fresh state** - Characters start with default level 1 stats and empty inventory -- **Automatic reset** - If username exists, it's reset to default state first -- **Works with all options** - Combine with location, combat levels, equipment, etc. - -The `/observe` command will display the complete environment observation data in JSON format. - -## Task Runner System - -AgentWorld includes a comprehensive task runner system that allows you to execute structured tasks from YAML configuration files. This is ideal for automated testing, benchmarking, and running complex multi-step scenarios. - -### Prerequisites for Task Runner - -Before using the task runner, ensure you have: - -1. **Game server running**: Start AgentWorld with `yarn dev` -2. **API keys configured**: Set up your LLM provider API keys as environment variables - -```bash -# For Qwen (Alibaba Cloud DashScope) -export DASHSCOPE_API_KEY=your-dashscope-api-key - -# For OpenAI -export OPENAI_API_KEY=sk-your-openai-key - -# For Anthropic Claude -export ANTHROPIC_API_KEY=sk-ant-your-claude-key - -# For DeepSeek -export DEEPSEEK_API_KEY=sk-your-deepseek-key -``` - -### Quick Start with Task Runner - -1. **Create a task configuration** (see `data_v0.1_solo/task_1.yaml` for example) -2. **Use an agent configuration** (see `agents/configs/qwen_agent.yaml`) -3. **Run the task**: - -```bash -# Run a single task -python3 agents/run.py --task data_v0.1_solo/task_1.yaml --agent agents/configs/qwen_agent.yaml --output logs/ - -# Run all tasks in a folder -python3 agents/run.py --task-folder data_v0.1_solo/ --agent agents/configs/qwen_agent.yaml --output logs/ -``` - -### Task Configuration Format - -Create YAML files that define structured tasks for your agents: - -```yaml -# Task definition -task: - name: "Smithing Task - Craft Heavy Sword" - description: "craft a heavy sword using smithing skills" - -# Task objectives -objectives: - primary: "Craft a heavy sword using smithing skills" - secondary: - - "Gather any missing materials if needed" - - "Use smithing station to craft the sword" - - "Verify successful crafting completion" - -# Agent configuration -agent_1: - username: "smith_test" - new_character: true - - location: - x: 88 - y: 33 - - skill_levels: - smithing: 15 - mining: 10 - - inventory_items: - - item: "ironbar" - count: 3 - - item: "hilt2" - count: 1 - - equipped_items: - - item: "smithinghammer" - count: 1 - enchant: 0 - -# Success criteria -success_criteria: - - "Heavy sword is successfully crafted" - -# Optional: Game context and limitations -max_action_steps: 100 -relevant_game_context: | - You are a smithing expert who knows how to use the forge and anvil effectively. -``` - -### Agent Configuration - -Agent configurations define the LLM provider settings and behavior: - -```yaml -# agents/configs/qwen_agent.yaml -agent: - name: "QwenAgent" - provider: "qwen" - - llm: - model: "qwen-plus" - api_key: null # Uses DASHSCOPE_API_KEY environment variable - base_url: "https://dashscope.aliyuncs.com/compatible-mode/v1" - max_tokens: 4096 - temperature: 0.7 - - game: - host: "http://localhost:7031" - timeout: 30 - max_retries: 3 - - # Custom system prompt template (optional) - system_prompt: | - You are an intelligent AI agent that plays the AgentWorld MMORPG game... - - === CURRENT ENVIRONMENT OBSERVATION === - {{observation}} - === END OBSERVATION === -``` - -### Key Features - -- **๐Ÿ“‹ Structured Task Definitions**: Define complex multi-step tasks in YAML -- **๐Ÿค– Multi-Agent Support**: Run tasks with multiple agents (agent_1, agent_2, etc.) -- **๐Ÿ”ง Configurable Agents**: Customize LLM provider, model, and system prompts -- **๐Ÿ“Š Comprehensive Logging**: Detailed logs saved to specified output folder -- **๐Ÿ“ˆ Execution Metrics**: Track duration, iterations, success rates -- **๐ŸŽฏ Batch Processing**: Run entire folders of tasks automatically - -### Output and Logging - -The task runner creates comprehensive logs in your specified output directory: - -``` -logs/ -โ”œโ”€โ”€ task_runner_20240115_143022.log # Main runner log -โ”œโ”€โ”€ agent_1_20240115_143022.json # Individual agent execution log -โ”œโ”€โ”€ task_summary_20240115_143022.json # Task execution summary -โ””โ”€โ”€ folder_summary_20240115_143022.json # Batch execution summary (if running folder) -``` - -### Advanced Usage - -```bash -# Run with different agent configurations -python3 agents/run.py --task task.yaml --agent configs/openai_agent.yaml --output results/ -python3 agents/run.py --task task.yaml --agent configs/claude_agent.yaml --output results/ - -# Run multiple tasks with comprehensive logging -python3 agents/run.py --task-folder experiments/ --agent configs/qwen_agent.yaml --output experiment_results/ -``` - -### Task Runner vs Console - -| Feature | Console (`console.py`) | Task Runner (`run.py`) | -|---------|----------------------|------------------------| -| **Use Case** | Interactive development, testing | Automated execution, benchmarking | -| **Input** | Command line arguments | YAML configuration files | -| **Output** | Console + optional JSON log | Structured logs + summaries | -| **Multi-agent** | Single agent per session | Multiple agents per task | -| **Reproducibility** | Manual setup each time | Fully reproducible from configs | -| **Batch Processing** | No | Yes (entire folders) | - -Choose **console.py** for interactive development and testing, and **run.py** for automated task execution and research experiments. - -### Troubleshooting - -#### Common Issues - -**API Key Errors (401 Unauthorized)** -``` -Error: API call failed: 401 Client Error: Unauthorized -``` -- Verify your API key is correctly set as an environment variable -- For Qwen: Check your DashScope API key format and permissions -- For other providers: Ensure the API key has sufficient credits/permissions - -**Game Server Connection Issues** -``` -Error: Game server not responding -``` -- Ensure AgentWorld is running with `yarn dev` -- Check that the game server is accessible at `http://localhost:7031` -- Verify no firewall is blocking the connection - -**Task Execution Failures** -- Check the detailed logs in the output directory -- Review the task summary JSON for specific error messages -- Ensure task configuration YAML syntax is valid - -#### Debug Commands - -```bash -# Test game server connectivity -curl http://localhost:7031/ai/observe - -# Test API key configuration -echo $DASHSCOPE_API_KEY # Should show your API key - -# Run with verbose logging -python3 agents/run.py --task task.yaml --agent config.yaml --output logs/ 2>&1 | tee debug.log -``` - -## License - -AgentWorld inherits the MPL-2.0 license from Kaetram. \ No newline at end of file +AgentWorld is a 2D multiplayer environment for evaluating how AI agents act, +communicate, and collaborate. This repository contains the game engine, Python +agent harness, benchmark tasks, and evaluation utilities. + +[Website and leaderboard](https://agentworld.io) ยท +[Submit results](https://agentworld.io/submission-guide) ยท +[Contributing](CONTRIBUTING.md) + +## Start here + +| Goal | Guide | +| --- | --- | +| Run your model on a task, then a benchmark suite | [Benchmark quickstart](docs/benchmark/quickstart.md) | +| Integrate your own agent harness or model adapter | [Custom agents](docs/benchmark/custom-agents.md) | +| Calculate scores and prepare a submission | [Scoring and results](docs/benchmark/scoring.md) | +| Understand where code belongs | [Repository map](docs/repository-map.md) | +| Start or configure the game server | [Server setup](docs/benchmark/server.md) | + +## Repository overview + +```text +agents/ Python runner, reference agents, tools, example configs +benchmarks/ Benchmark entry-point documentation and offline SR reporting +data_v0.1_multi/ Versioned multi-agent task definitions and verifiers +data_v0.1_solo/ Solo task definitions +examples/agents/ Custom-agent adapter example +analysis/ Research analysis and CCE judge scripts +packages/ TypeScript game server, client, shared code, and tools +docs/ Onboarding, protocol notes, and game documentation +tests/benchmark/ Offline regression tests for benchmark entry points +``` + +The quickstart uses `data_v0.1_multi/v1.3_benchmark` (100 main tasks) and +`data_v0.1_multi/v1.3_augmented` (200 variants). Older task versions and +machine-specific launch scripts remain available for reproducing historical work; +see the repository map before using them. Scoring limitations are documented +explicitly; a successful local run does not by itself certify leaderboard comparability. + +## Credits and license + +Built upon [Kaetram](https://github.com/Kaetram/Kaetram-Open), which expands on +Little Workshop's BrowserQuest. Licensed under [MPL-2.0](LICENSE). + +Detailed historical console examples are retained in [Legacy usage](docs/legacy-usage.md). diff --git a/agents/README.md b/agents/README.md index f80faf89b..f1e55b9c0 100644 --- a/agents/README.md +++ b/agents/README.md @@ -1,233 +1,30 @@ -# Qwen AI Agent for Kaetram Game - -An intelligent AI agent that plays the Kaetram MMORPG game using Alibaba Cloud Qwen Function Calling capabilities. - -## Features - -- **OpenAI Compatible Function Calling**: Uses standard function calling format without XML dependencies -- **Clean Implementation**: No custom XML parsing - relies on native API function calling -- **Intelligent Gameplay**: AI makes strategic decisions about movement, combat, and resource collection -- **Interactive Mode**: Chat with the agent and give it specific instructions -- **Auto-Play Mode**: Let the agent play autonomously -- **Full Game API Coverage**: Supports all Kaetram AI agent APIs - -## Prerequisites - -1. **Kaetram Game Server**: Must be running on `http://localhost:9002` -2. **Alibaba Cloud Dashscope API Key**: Required for Qwen model access -3. **Python 3.8+**: Required for running the agent - -## Quick Start - -### 1. Install Dependencies - -```bash -cd agents/qwen -pip install -r requirements.txt -``` - -### 2. Configure API Key - -Set your Dashscope API key as an environment variable: - -```bash -export DASHSCOPE_API_KEY=your-actual-api-key -``` - -Or modify the `config.py` file directly. - -### 3. Start Kaetram Game Server - -Make sure the Kaetram game server is running with API enabled: - -```bash -# In the main Kaetram directory -yarn dev -``` - -The server should be accessible at `http://localhost:9002` - -### 4. Run the Agent - -#### Interactive Mode (Default) -```bash -python main.py -``` - -#### Auto-Play Mode -```bash -python main.py --auto -python main.py --auto --steps 20 # Run for 20 steps -``` - -#### Test Connections -```bash -python main.py --test -``` - -#### Run Examples -```bash -python example.py # Run all usage examples -python spawn_example.py # Demo automatic spawn position teleport -``` - -## Usage Examples - -### Interactive Mode Commands - -- `start` - Start a new game session (login/create character) -- `auto` - Switch to auto-play mode -- `reset` - Reset conversation history -- `quit` or `exit` - Exit the program - -### Sample Interactions - -``` -๐Ÿ‘ค You: start -๐Ÿค– Agent: [Logs into the game and starts playing] - -๐Ÿ‘ค You: explore the area and tell me what you see -๐Ÿค– Agent: [Observes environment and describes surroundings] - -๐Ÿ‘ค You: find some resources and collect them -๐Ÿค– Agent: [Searches for and collects available resources] - -๐Ÿ‘ค You: if you see any enemies, attack them -๐Ÿค– Agent: [Engages in combat if enemies are found] -``` - -## Configuration - -Edit `config.py` to customize: - -- **API Settings**: Qwen model, API endpoints -- **Game Settings**: Server URL, character credentials -- **Agent Behavior**: Combat enabled, chat enabled, exploration priority -- **Spawn Position**: Automatic teleport to specific coordinates after login - -### Spawn Position Configuration - -Configure automatic teleportation after login in `config.py`: - -```python -SPAWN_POSITION = { - "enabled": True, # Set to False to disable auto teleport - "x": 250, # X coordinate for spawn position - "y": 180, # Y coordinate for spawn position - "withAnimation": False # Whether to show teleport animation -} -``` - -When enabled, the agent will automatically teleport to the specified coordinates immediately after successful login or character creation. - -## Game Actions Supported - -The agent can perform all Kaetram AI actions: - -1. **Character Management** - - Create character - - Login character - -2. **Movement & Navigation** - - Move to coordinates - - Enter portals - - Stop movement - -3. **Environment Interaction** - - Observe surroundings - - Collect resources - - Target entities - -4. **Combat System** - - Target enemies - - Attack targets - - Stop combat - -5. **Equipment & Inventory** - - Equip items - - Manage inventory - -6. **Social Features** - - Send chat messages - - Interact with players - -## Architecture - -``` -agents/qwen/ -โ”œโ”€โ”€ config.py # Configuration settings -โ”œโ”€โ”€ game_tools.py # Kaetram API wrapper functions -โ”œโ”€โ”€ tool_definitions.py # Function calling tool definitions -โ”œโ”€โ”€ qwen_agent.py # Main AI agent implementation -โ”œโ”€โ”€ main.py # Entry point and CLI interface -โ”œโ”€โ”€ example.py # Usage examples and demos -โ”œโ”€โ”€ spawn_example.py # Spawn position demo -โ”œโ”€โ”€ requirements.txt # Python dependencies -โ””โ”€โ”€ README.md # This file -``` - -## How It Works - -1. **OpenAI Compatible Function Calling**: Uses standard OpenAI function calling format via tools parameter -2. **Clean Implementation**: No XML tags or custom parsing - relies on native API function calling -3. **API Integration**: Game tools wrap Kaetram's REST APIs with proper authentication -4. **Auto Spawn Positioning**: Automatically teleports to configured coordinates after successful login -5. **Strategic Decision Making**: The AI model makes intelligent decisions based on game state -6. **Continuous Learning**: The agent maintains conversation context for better decision making - -## Troubleshooting - -### Common Issues - -1. **API Key Error** - ``` - Error: DASHSCOPE_API_KEY not configured! - ``` - Solution: Set your Dashscope API key in environment variables - -2. **Game Server Connection Failed** - ``` - Game server connection failed: Connection refused - ``` - Solution: Ensure Kaetram server is running on localhost:9002 - -3. **Character Already Exists** - ``` - Failed to create character: Username already exists - ``` - Solution: The agent will automatically try to login instead - -4. **Token Errors** - ``` - Error: No token available. Please login first. - ``` - Solution: Run the 'start' command to login first - -### Debug Mode - -Enable debug logging by modifying `config.py`: - -```python -DEBUG_MODE = True -VERBOSE_LOGGING = True -``` - -## API Reference - -For detailed information about Kaetram's AI Agent APIs, see: -- [Kaetram AI API Documentation](../server/src/network/README-AI-API.md) - -For Qwen Function Calling documentation, see: -- [Alibaba Cloud Qwen Function Calling](https://www.alibabacloud.com/help/zh/model-studio/qwen-function-calling) - -## Contributing - -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Test with both interactive and auto-play modes -5. Submit a pull request - -## License - -This project follows the same license as the main Kaetram project. \ No newline at end of file +# Python agents and benchmark runner + +Start with the [benchmark quickstart](../docs/benchmark/quickstart.md). + +| File | Responsibility | +| --- | --- | +| `run.py` | Task initialization, round scheduling, trajectories, early stopping | +| `config_loader.py` | YAML and explicit environment-variable references | +| `agent_factory.py` | Built-in providers and custom adapter loading | +| `base_agent.py` | Shared prompts, tool execution, and one-action-per-turn contract | +| `*_agent.py` | Provider-specific model requests and response conversion | +| `game_tools.py`, `tool_definitions.py` | Game API client and agent-visible actions | +| `experiment.py` | Baselines, round budgets, communication and spawn controls | +| `verification.py` | Task-specific verifier selection shared with offline scoring | +| `console.py` | Interactive debugging and character initialization | + +Use `configs/openai-compatible.example.yaml` for an HTTP model endpoint, or +`configs/custom.example.yaml` with the [custom-agent guide](../docs/benchmark/custom-agents.md). +Install `requirements.txt` for API models; `requirements-local.txt` additionally +installs vLLM for the `deepseek_local` provider. + +The `run_*.sh` and rerun scripts are historical experiment launchers, often with +absolute paths and machine-specific settings. Use `python agents/run.py --help` +from the repository root as the portable entry point. + +See [BASELINES.md](BASELINES.md) for experiment variants. Record all non-default +flags when reporting results. + +The earlier provider-specific documentation is retained in the +[historical Qwen guide](../docs/legacy-qwen-agent.md). diff --git a/agents/agent_factory.py b/agents/agent_factory.py index ddf5d9e41..d093d54ed 100644 --- a/agents/agent_factory.py +++ b/agents/agent_factory.py @@ -3,6 +3,7 @@ Provides a unified interface to create agents for different LLM providers """ +import importlib from typing import Optional from base_agent import BaseAgent from qwen_agent import QwenAgent @@ -16,6 +17,7 @@ class AgentFactory: """Factory class for creating different types of LLM agents""" SUPPORTED_PROVIDERS = { + "custom": "User-supplied BaseAgent adapter", "qwen": "Alibaba Cloud Qwen (DashScope)", "openai": "OpenAI (GPT-4, GPT-3.5, etc.)", "claude": "Anthropic Claude", @@ -62,6 +64,19 @@ def create_agent( supported = ", ".join(cls.SUPPORTED_PROVIDERS.keys()) raise ValueError(f"Unsupported provider: {provider}. Supported providers: {supported}") + if provider == "custom": + # Configured Python adapters execute trusted local code. + target = llm_params.get("agent_class", "") + module_name, separator, class_name = target.partition(":") + if not separator or not module_name or not class_name: + raise ValueError("Custom provider requires llm.agent_class: module:Class") + agent_class = getattr(importlib.import_module(module_name), class_name) + if not isinstance(agent_class, type) or not issubclass(agent_class, BaseAgent): + raise ValueError("Custom agent class must inherit BaseAgent") + return agent_class(api_key=api_key, model=model, username=username, + password=password, base_url=base_url, + dump_prompts=dump_prompts, llm_params=llm_params) + if provider == "qwen": return QwenAgent(username=username, password=password, base_url=base_url, dump_prompts=dump_prompts) @@ -69,7 +84,7 @@ def create_agent( if not api_key: raise ValueError("OpenAI API key is required for OpenAI provider") default_model = model or "gpt-4o" - return OpenAIAgent(api_key=api_key, model=default_model, username=username, password=password, base_url=base_url, dump_prompts=dump_prompts) + return OpenAIAgent(api_key=api_key, model=default_model, username=username, password=password, base_url=base_url, dump_prompts=dump_prompts, llm_params=llm_params) elif provider == "claude": if not api_key: @@ -151,7 +166,7 @@ def validate_provider_config(cls, provider: str, api_key: Optional[str] = None) return True # Local DeepSeek runs fully offline - if provider == "deepseek_local": + if provider in ("deepseek_local", "custom"): return True # All other providers require API keys diff --git a/agents/config_loader.py b/agents/config_loader.py new file mode 100644 index 000000000..f04a5f88b --- /dev/null +++ b/agents/config_loader.py @@ -0,0 +1,29 @@ +"""Load YAML configuration, resolving explicit ${ENV_VAR} values after parsing.""" +import os +import re +from pathlib import Path + +import yaml + +_ENV = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def load_agent_config(path): + def resolve(value): + if isinstance(value, dict): + return {key: resolve(item) for key, item in value.items()} + if isinstance(value, list): + return [resolve(item) for item in value] + if isinstance(value, str): + def substitute(match): + name = match.group(1) + if not os.environ.get(name): + raise ValueError(f"Set the non-empty environment variable {name}") + return os.environ[name] + return _ENV.sub(substitute, value) + return value + + data = resolve(yaml.safe_load(Path(path).read_text())) + if not isinstance(data, dict) or not isinstance(data.get("agent"), dict): + raise ValueError("Configuration must contain an agent mapping") + return data diff --git a/agents/configs/custom.example.yaml b/agents/configs/custom.example.yaml new file mode 100644 index 000000000..bfd609ab1 --- /dev/null +++ b/agents/configs/custom.example.yaml @@ -0,0 +1,12 @@ +# Example adapter: PYTHONPATH=examples/agents python agents/run.py ... +agent: + name: MyHarness + provider: custom + llm: + agent_class: custom_agent:CustomAgent + model: ${MODEL_NAME} + api_key: ${MODEL_API_KEY} + base_url: ${MODEL_BASE_URL} + auth_header: Authorization + game: + host: ${AGENTWORLD_BASE_URL} diff --git a/agents/configs/openai-compatible.example.yaml b/agents/configs/openai-compatible.example.yaml new file mode 100644 index 000000000..831c0b4f7 --- /dev/null +++ b/agents/configs/openai-compatible.example.yaml @@ -0,0 +1,11 @@ +# Export these variables in your shell; the runner resolves ${NAME} after YAML parsing. +agent: + name: MyModel + provider: openai + llm: + model: ${MODEL_NAME} + api_key: ${MODEL_API_KEY} + base_url: ${MODEL_BASE_URL} + auth_header: Authorization + game: + host: ${AGENTWORLD_BASE_URL} diff --git a/agents/openai_agent.py b/agents/openai_agent.py index 519ac04f0..85b5a1ae6 100644 --- a/agents/openai_agent.py +++ b/agents/openai_agent.py @@ -10,18 +10,24 @@ class OpenAIAgent(BaseAgent): - def __init__(self, api_key: str, model: str = "gpt-4o", username: Optional[str] = None, password: Optional[str] = None, base_url: Optional[str] = None, dump_prompts: bool = False): + def __init__(self, api_key: str, model: str = "gpt-4o", username: Optional[str] = None, password: Optional[str] = None, base_url: Optional[str] = None, dump_prompts: bool = False, llm_params: Optional[dict] = None): # Initialize parent class super().__init__(username, password, base_url, dump_prompts) # OpenAI-specific configuration self.api_key = api_key or 'agentworld' self.model = model - self.base_url = "https://model-gateway.acenta.ai/v1" + llm_params = llm_params or {} + # Preserve existing gateway configs; explicit endpoints use Bearer auth. + self.base_url = llm_params.get("base_url", "https://model-gateway.acenta.ai/v1").rstrip("/") + auth_header = llm_params.get("auth_header", "Authorization" if "base_url" in llm_params else "X-API-Key") + if auth_header not in ("Authorization", "X-API-Key"): + raise ValueError("auth_header must be Authorization or X-API-Key") + auth_value = f"Bearer {self.api_key}" if auth_header == "Authorization" else self.api_key self.provider = "openai" self.session = requests.Session() self.session.headers.update({ - "X-API-Key": self.api_key, + auth_header: auth_value, "Content-Type": "application/json" }) diff --git a/agents/requirements-local.txt b/agents/requirements-local.txt new file mode 100644 index 000000000..d6b342d02 --- /dev/null +++ b/agents/requirements-local.txt @@ -0,0 +1,3 @@ +# Optional in-process GPU inference; not needed for an HTTP model endpoint. +-r requirements.txt +vllm>=0.6.2 diff --git a/agents/requirements.txt b/agents/requirements.txt index ead2a156c..85485d1da 100644 --- a/agents/requirements.txt +++ b/agents/requirements.txt @@ -1,27 +1,4 @@ -# Core HTTP and networking +# API-based benchmark runner (CPU-only installation). requests>=2.31.0 -urllib3>=2.0.0 -certifi>=2023.0.0 -charset-normalizer>=3.0.0 -idna>=3.0 - -# Optional: Official LLM provider SDKs (comment out if using requests only) -# openai>=1.0.0 # Official OpenAI Python client -# anthropic>=0.25.0 # Official Anthropic Claude client - -# Local inference (DeepSeek via vLLM) -vllm>=0.6.2 - -# YAML parsing for task and config files PyYAML>=6.0.0 - -# JSON and data handling (usually included with Python) -# json is built-in - -# Template engine for prompt formatting Jinja2>=3.1.0 - -# Development and testing (optional) -# pytest>=7.0.0 -# black>=23.0.0 -# flake8>=6.0.0 \ No newline at end of file diff --git a/agents/run.py b/agents/run.py index e1cd1454b..61966bac3 100755 --- a/agents/run.py +++ b/agents/run.py @@ -35,6 +35,8 @@ # Import existing agent components from agent_factory import AgentFactory +from config_loader import load_agent_config +from verification import verify_trajectory from console import GameConsole from config import MASTER_PASSWORD from base_agent import BaseAgent @@ -414,8 +416,7 @@ def __init__(self, agent_config_path: str, output_dir: str, use_split_screen: bo def _load_agent_config(self) -> AgentConfig: """Load agent configuration from YAML file""" try: - with open(self.agent_config_path, 'r') as f: - config_data = yaml.safe_load(f) + config_data = load_agent_config(self.agent_config_path) agent_data = config_data['agent'] return AgentConfig( @@ -1886,6 +1887,8 @@ def _init_trajectory(self, task_path: str, task_config: TaskConfig): self.trajectory_data = { 'task_id': f"task_{task_number}", + 'task_source': str(Path(task_path).resolve()), + 'task_key': task_filename, 'timestamp': datetime.now().isoformat(), 'task_definition': task_definition, 'rounds': [], @@ -1959,7 +1962,11 @@ def _save_trajectory(self, task_config: TaskConfig, results: Dict[str, Any]): """Save trajectory data to JSON file""" # Extract task number from task_id task_number = self.trajectory_data['task_id'].split('_')[1] if '_' in self.trajectory_data['task_id'] else self.trajectory_data['task_id'] - trajectory_file = self.output_dir / f"task_{task_number}_trajectory.json" + # Keep the historical main-task filename; variants need distinct files. + task_key = self.trajectory_data.get('task_key', f'task_{task_number}') + suffix = task_key.rsplit('_', 1)[-1] + filename = task_key if suffix in ('v1', 'v2') else f'task_{task_number}' + trajectory_file = self.output_dir / f"{filename}_trajectory.json" # Update metrics from results if '_multi_agent_metrics' in results: @@ -1979,48 +1986,7 @@ def _check_task_success_early(self) -> Tuple[bool, str]: Tuple[bool, str]: (success, message) where success is True if task is complete """ try: - # Get task number from trajectory data - task_id = self.trajectory_data.get('task_id', '') - if not task_id: - return False, "No task_id in trajectory data" - - # Extract task number (e.g., "task_01" -> "01") - import re - match = re.search(r'task_(\d+)', task_id) - if not match: - return False, f"Could not parse task number from {task_id}" - - task_num = int(match.group(1)) - - # Try to import the verifier module - verifier_path = Path(__file__).parent.parent / "data_v0.1_multi" / "v1.3_benchmark" - verifier_file = verifier_path / f"task_{task_num:02d}_success_criteria.py" - - if not verifier_file.exists(): - self.logger.debug(f"No verifier found at {verifier_file}") - return False, f"No verifier for task {task_num:02d}" - - # Dynamically import the verifier module - import importlib.util - spec = importlib.util.spec_from_file_location(f"task_{task_num:02d}_verifier", verifier_file) - verifier_module = importlib.util.module_from_spec(spec) - - # Add verifier_utils to the module's namespace - verifier_utils_path = verifier_path / "verifier_utils.py" - if verifier_utils_path.exists(): - utils_spec = importlib.util.spec_from_file_location("verifier_utils", verifier_utils_path) - utils_module = importlib.util.module_from_spec(utils_spec) - sys.modules['verifier_utils'] = utils_module - utils_spec.loader.exec_module(utils_module) - - spec.loader.exec_module(verifier_module) - - # Call the verify function - if hasattr(verifier_module, 'verify'): - success, msg = verifier_module.verify(self.trajectory_data) - return bool(success), msg - else: - return False, "Verifier module has no verify() function" + return verify_trajectory(self.trajectory_data) except Exception as e: self.logger.debug(f"Early success check failed: {e}") diff --git a/agents/verification.py b/agents/verification.py new file mode 100644 index 000000000..de69fd472 --- /dev/null +++ b/agents/verification.py @@ -0,0 +1,53 @@ +"""Shared verifier selection for runner early stopping and offline SR scoring.""" +import importlib.util +import re +import sys +from functools import lru_cache +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MAIN = ROOT / 'data_v0.1_multi/v1.3_benchmark' + + +def verifier_for_task(task_path): + task_path = Path(task_path) + adjacent = task_path.with_name(task_path.stem + '_success_criteria.py') + if adjacent.is_file(): + return adjacent + # Main tasks use task_NN_success_criteria.py rather than the full task name. + match = re.fullmatch(r'task_(\d+)_.+', task_path.stem) + if task_path.parent.resolve() == MAIN and match: + verifier = MAIN / f'task_{int(match[1]):02d}_success_criteria.py' + if verifier.is_file(): + return verifier + raise ValueError(f'No task-specific verifier for {task_path.name}') + + +@lru_cache(maxsize=None) +def _load_verifier(path): + if 'verifier_utils' not in sys.modules: + spec = importlib.util.spec_from_file_location('verifier_utils', MAIN / 'verifier_utils.py') + module = importlib.util.module_from_spec(spec) + sys.modules['verifier_utils'] = module + spec.loader.exec_module(module) + spec = importlib.util.spec_from_file_location('agentworld_' + path.stem, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.verify + + +def verify_trajectory(trajectory, task_path=None): + # Explicit catalog paths take precedence over paths saved on other machines. + source = task_path or trajectory.get('task_source') + if source: + verifier = verifier_for_task(source) + else: + # Compatibility with historical main-task logs lacking source metadata. + match = re.fullmatch(r'task_(\d+)', trajectory.get('task_id', '')) + if not match: + raise ValueError('A task source is required to select the verifier') + verifier = MAIN / f'task_{int(match[1]):02d}_success_criteria.py' + if not verifier.is_file(): + raise ValueError('Unknown task ID') + passed, details = _load_verifier(verifier)(trajectory) + return bool(passed), details diff --git a/analysis/requirements-cce.txt b/analysis/requirements-cce.txt new file mode 100644 index 000000000..979de7050 --- /dev/null +++ b/analysis/requirements-cce.txt @@ -0,0 +1,2 @@ +# Optional LLM judge client for compute_cce_v2.py +openai>=1.0.0 diff --git a/analysis/scripts/compute_cce_v2.py b/analysis/scripts/compute_cce_v2.py index 6f145c600..72f6c9a84 100644 --- a/analysis/scripts/compute_cce_v2.py +++ b/analysis/scripts/compute_cce_v2.py @@ -18,10 +18,10 @@ import argparse from openai import OpenAI -# Config -API_KEY = "sk-QJ3tm90vbicqSOMEW5auku5OeR6cvbHo59yy7mRNH5hxCuTz" -BASE_URL = "https://yinli.one/v1" -JUDGE_MODEL = "gpt-4.1" +# Configure the judge explicitly. Importing this module makes no API calls. +API_KEY = os.environ.get("CCE_API_KEY", "") +BASE_URL = os.environ.get("CCE_BASE_URL", "https://api.openai.com/v1") +JUDGE_MODEL = os.environ.get("CCE_JUDGE_MODEL", "gpt-4.1") MODEL_DIRS = { 'gemini': 'gemini_v1.3', @@ -33,9 +33,9 @@ 'deepseek_aug': 'deepseek_augmented', 'gemini_aug': 'gemini_augmented', } -LOGS_BASE = os.path.expanduser('~/works/agentworld/agents/logs') +LOGS_BASE = os.environ.get('AGENTWORLD_LOGS_DIR', os.path.join(os.path.dirname(__file__), '../../agents/logs')) -client = OpenAI(api_key=API_KEY, base_url=BASE_URL) +client = None def call_llm(prompt, verbose=False): @@ -46,6 +46,11 @@ def call_llm(prompt, verbose=False): print(prompt[:500] + "..." if len(prompt) > 500 else prompt) print(f"{'โ”€'*60}") + global client + if client is None: + if not API_KEY: + raise ValueError("Set CCE_API_KEY before computing CCE") + client = OpenAI(api_key=API_KEY, base_url=BASE_URL) response = client.chat.completions.create( model=JUDGE_MODEL, max_tokens=2000, @@ -327,6 +332,7 @@ def compute_cce_v2(trajectory, verbose=False): def main(): parser = argparse.ArgumentParser() + parser.add_argument('--trajectory', help='Score one trajectory JSON from any run directory') parser.add_argument('--model', type=str, default='gemini', choices=MODEL_DIRS.keys()) parser.add_argument('--task', type=str, default='1', help='Task number or "all" for batch run') parser.add_argument('--verbose', action='store_true', help='Print detailed LLM interactions') @@ -336,7 +342,11 @@ def main(): model_dir = os.path.join(LOGS_BASE, MODEL_DIRS[args.model]) # Determine task list - if args.task == 'all': + if args.trajectory: + if not os.path.isfile(args.trajectory): + parser.error('--trajectory does not exist') + tasks = [(0, args.trajectory)] + elif args.task == 'all': traj_files = sorted(glob.glob(os.path.join(model_dir, 'task_*_trajectory.json'))) tasks = [] for f in traj_files: @@ -351,7 +361,8 @@ def main(): return tasks = [(task_num, filepath)] - print(f"CCE v2 | Model: {args.model} | Judge: {JUDGE_MODEL} | Tasks: {len(tasks)}") + run_label = 'trajectory' if args.trajectory else args.model + print(f"CCE v2 | Model: {run_label} | Judge: {JUDGE_MODEL} | Tasks: {len(tasks)}") print(f"{'='*70}") all_results = [] @@ -395,11 +406,13 @@ def main(): print(f"Avg CE (all tasks): {avg_ce_all:.4f}") # Save results - output_path = args.output or f'results/cce_v2_{args.model}_{JUDGE_MODEL}.json' + output_path = args.output or f'results/cce_v2_{run_label}_{JUDGE_MODEL}.json' os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) with open(output_path, 'w') as f: json.dump({ - 'model': args.model, + 'model': None if args.trajectory else args.model, + 'trajectory': args.trajectory, + 'judge_base_url': BASE_URL, 'judge': JUDGE_MODEL, 'results': all_results, 'summary': { diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..6ba959010 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,10 @@ +# Benchmark entry points + +1. [Run a model](../docs/benchmark/quickstart.md) with `agents/run.py`. +2. [Integrate a harness](../docs/benchmark/custom-agents.md) through `BaseAgent`. +3. [Score and submit](../docs/benchmark/scoring.md) with `benchmarks/score.py`. + +Task definitions remain in their existing versioned directories. This directory +provides a discoverable entry point without moving datasets or changing task +criteria. `score.py` performs offline success-rate reporting only; it makes no +model API calls. diff --git a/benchmarks/score.py b/benchmarks/score.py new file mode 100644 index 000000000..6a92ba0a2 --- /dev/null +++ b/benchmarks/score.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Offline SR from repository task verifiers; does not calculate PSR or CCE.""" +import argparse +import json +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'agents')) +from verification import verify_trajectory + +SUITES = { + 'main': ROOT / 'data_v0.1_multi/v1.3_benchmark', + 'augmented': ROOT / 'data_v0.1_multi/v1.3_augmented', +} + + +def task_key(trajectory, catalog): + key = trajectory.get('task_key') + if not key and trajectory.get('task_source'): + key = Path(trajectory['task_source']).stem + if key: + if key not in catalog: + raise ValueError(f'Task {key} is not in the selected suite') + return key + # Old main logs have only task_NN. Never guess between augmented variants. + identifier = trajectory.get('task_id', '') + if not re.fullmatch(r'task_\d+', identifier): + raise ValueError('Missing task_key/task_source and invalid task_id') + matches = [k for k in catalog if k.startswith(identifier + '_')] + if len(matches) != 1: + raise ValueError(f'Ambiguous or unknown task_id {identifier}; supply task_key') + return matches[0] + + +def score(directory, suite): + catalog = {p.stem: p for p in SUITES[suite].glob('*.yaml')} + if not catalog: + raise ValueError('Selected suite has no tasks') + rows, errors, seen = [], [], set() + for path in sorted(Path(directory).rglob('*_trajectory.json')): + try: + trajectory = json.loads(path.read_text()) + key = task_key(trajectory, catalog) + if key in seen: + raise ValueError(f'Duplicate trajectory for {key}; score one trial at a time') + seen.add(key) + if not isinstance(trajectory.get('rounds'), list) or not trajectory['rounds']: + raise ValueError('No recorded rounds; incomplete or invalid trajectory') + passed, details = verify_trajectory(trajectory, catalog[key]) + rows.append({'task': key, 'success': passed, 'details': details, 'file': str(path)}) + except Exception as error: + errors.append({'file': str(path), 'error': str(error)}) + missing = sorted(set(catalog) - {r['task'] for r in rows}) + complete = not missing and not errors + successes = sum(row['success'] for row in rows) + return { + 'suite': suite, 'metric': 'repository_verifier_SR', + 'expected_tasks': len(catalog), 'scored_tasks': len(rows), + 'successes': successes, 'complete': complete, + 'SR_percent': 100 * successes / len(catalog) if complete else None, + 'observed_SR_percent': 100 * successes / len(rows) if rows and not errors else None, + 'missing_tasks': missing, 'errors': errors, 'results': rows, + 'note': 'Uses this checkout\'s task verifiers. Not a certification of paper-protocol equivalence.', + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--suite', choices=SUITES, required=True) + parser.add_argument('--trajectories', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if not args.trajectories.is_dir(): + parser.error('--trajectories must be an existing directory') + report = score(args.trajectories, args.suite) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps({k: report[k] for k in ('suite', 'scored_tasks', 'expected_tasks', 'complete', 'SR_percent', 'observed_SR_percent')}, indent=2)) + if report['errors'] or not report['scored_tasks']: + print(f"Scoring needs attention; see {args.output}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs/benchmark/custom-agents.md b/docs/benchmark/custom-agents.md new file mode 100644 index 000000000..d1bd91cd6 --- /dev/null +++ b/docs/benchmark/custom-agents.md @@ -0,0 +1,68 @@ +# Integrating a model or agent harness + +## A model behind an OpenAI-compatible endpoint + +Use `agents/configs/openai-compatible.example.yaml` and the +[quickstart](quickstart.md). You do not need to modify the factory or game engine. +Keep the endpoint's model ID separate from the name you use on the leaderboard. + +## A custom Python adapter in the reference runner + +Set `agent.provider: custom` and `agent.llm.agent_class: module:Class`. The +factory imports that class from your Python path. Configs selecting custom classes +execute local Python code; use an adapter you trust. + +A runnable starting point is `examples/agents/custom_agent.py`. It inherits the +OpenAI-compatible policy unchanged so you can first verify wiring: + +```bash +# Set the same model/game variables as in the quickstart. +PYTHONPATH=examples/agents python agents/run.py \ + --task data_v0.1_multi/v1.3_benchmark/task_01_magic_staff.yaml \ + --agent agents/configs/custom.example.yaml \ + --output runs/my-harness/smoke \ + --no-split-screen +``` + +Your class must inherit `BaseAgent` and accept these constructor keywords: +`api_key`, `model`, `username`, `password`, `base_url`, `dump_prompts`, and +`llm_params`. `base_url` is the **game** URL; model settings live in `llm_params`. +A custom harness may omit `api_key` from its YAML if it does not require one. +The runner still requires `llm.model` as its configured model identifier. + +The integration points are: + +- `_make_api_call(messages)`: return an OpenAI-shaped response with + `choices[0].message` and its tool calls. See `agents/openai_agent.py`. +- `_extract_tool_calls(message)`: normalize calls to dictionaries with `id`, + `name`, and parsed `arguments`. See the existing provider adapters. +- `execute_single_tool_call(user_input)`: the runner calls this once per agent + turn. The inherited implementation executes one action, records conversation + history, and emits the action/result information used in trajectories. + +For a different planner, memory system, or agent framework, wrap it behind that +contract. Keep `game_tools`, tool definitions, experiment restriction flags, +and action recording intact. Overriding only `process_user_input()` does not +change the benchmark's round-based execution path. If you replace +`execute_single_tool_call`, preserve its one-action-per-turn behavior and the +`[TOOL_CALL_INFO]` / `[TOOL_RESULT]` records consumed by `agents/run.py`. + +Changing prompts or policy is a harness change: report it together with model +settings, action/round budget, source commit, and repetitions. Extra model calls +or hidden game actions must be disclosed; do not treat them as the reference +harness protocol. + +## An independent harness + +Use `agents/game_tools.py`, `agents/tool_definitions.py`, and the +[server API reference](../../packages/server/src/network/README-AI-API.md) as the +integration surface. The task YAML defines objectives, initial characters, +inventories, skill levels, and starting positions. + +Independent harnesses must reproduce that initialization and record the same +observations/actions needed by the task verifiers. The offline SR scorer expects +`*_trajectory.json` files containing `task_key` (the task YAML stem) and non-empty +`rounds`, with each round's `actions` including `agent_name` and `observation`. +This is a minimum identification shape, not the entire verifier contract: retain +all fields emitted by the reference runner and inspect the relevant task verifier. +Do not use a result-only JSON file as a substitute for the trajectory. diff --git a/docs/benchmark/quickstart.md b/docs/benchmark/quickstart.md new file mode 100644 index 000000000..c50eb89e2 --- /dev/null +++ b/docs/benchmark/quickstart.md @@ -0,0 +1,103 @@ +# Benchmark quickstart + +Run commands from the repository root. Start with one task before spending on a +full suite. This guide uses the reference harness; custom harness integration is +covered in [Custom agents](custom-agents.md). + +## 1. Install the Python runner + +Use Python 3.10 or newer (the offline onboarding checks use Python 3.14). + +```bash +git clone --branch develop https://github.com/openagents-org/agentworld.git +cd agentworld +python3 -m venv .venv +source .venv/bin/activate +python -m pip install -r agents/requirements.txt +python agents/run.py --help +``` + +API-based models do not require CUDA, PyTorch, or vLLM. To use the existing +in-process `deepseek_local` provider, install `agents/requirements-local.txt` +instead. An externally hosted OpenAI-compatible model server can use the normal +CPU-only runner installation. + +## 2. Start a game instance + +Follow [Server setup](server.md). The default game HTTP API is +`http://localhost:7031`; this is separate from the model inference endpoint. +Use an isolated game instance for each concurrent experiment: shared world state +and reused task character names can interfere with another run. + +## 3. Configure your model + +The example resolves `${NAME}` values from the current shell after parsing YAML. +Missing or empty variables fail before agent initialization. `.env` is used by +the game server; the Python runner does not automatically load it. + +```bash +export MODEL_NAME='your-tool-calling-model-id' +export MODEL_BASE_URL='http://localhost:8000/v1' +export MODEL_API_KEY='your-model-api-key' +export AGENTWORLD_BASE_URL='http://localhost:7031' +``` + +For a local endpoint without authentication, set `MODEL_API_KEY=unused`. +For OpenAI's API, set `MODEL_BASE_URL=https://api.openai.com/v1` and use your key +and a supported model ID. The adapter sends Chat Completions requests with +function tools, required tool choice, and `parallel_tool_calls=false`; your +endpoint must support that contract. It is not a Responses API adapter. + +The example's `llm.base_url` and `llm.auth_header` configure the model endpoint; +`game.host` configures AgentWorld. `Authorization` sends a Bearer token; +`X-API-Key` supports gateways that require that header. + +## 4. Run a smoke task + +```bash +python agents/run.py \ + --task data_v0.1_multi/v1.3_benchmark/task_01_magic_staff.yaml \ + --agent agents/configs/openai-compatible.example.yaml \ + --output runs/my-model/smoke \ + --no-split-screen +``` + +Inspect `runs/my-model/smoke/task_01_trajectory.json`: it should contain recorded +rounds, observations, actions, task metadata, and verification metrics. Task +initialization/model errors are not successful benchmark attempts. + +## 5. Run one complete trial + +Use a fresh output directory for every model, harness configuration, and trial. +The runner does not resume or protect a reused output directory from overwrites. + +```bash +python agents/run.py \ + --task-folder data_v0.1_multi/v1.3_benchmark \ + --agent agents/configs/openai-compatible.example.yaml \ + --output runs/my-model/main/trial-1 \ + --no-split-screen +``` + +The main directory contains 100 task YAMLs. For the 200 augmented variants, use +`data_v0.1_multi/v1.3_augmented` and a separate output directory, for example +`runs/my-model/augmented/trial-1`. Variant trajectories retain the full task name +including `_v1` or `_v2` so they do not overwrite each other. + +Do not use the parent `data_v0.1_multi/` directory as a suite. It contains several +historical datasets. Do not change tasks, starting state, prompts, tools, or round +budgets without reporting the resulting protocol change. See +[baselines](../../agents/BASELINES.md) for supported experiment flags. + +## 6. Score and submit + +```bash +python benchmarks/score.py \ + --suite main \ + --trajectories runs/my-model/main/trial-1 \ + --output runs/my-model/main/trial-1/scores.json +``` + +The report distinguishes coverage from success rate. See +[Scoring and results](scoring.md) for incomplete runs, CCE, PSR, repeated trials, +and the GitHub issue submission route. diff --git a/docs/benchmark/scoring.md b/docs/benchmark/scoring.md new file mode 100644 index 000000000..f191464c8 --- /dev/null +++ b/docs/benchmark/scoring.md @@ -0,0 +1,85 @@ +# Scoring and result submission + +Keep the model, harness, task suite, code revision, and trial count alongside every +result. Current repository verifiers are executable checks, not evidence that +every task criterion matches the paper perfectly. Protocol and verifier auditing +remain necessary for published comparisons. + +## Success rate (SR) + +```bash +python benchmarks/score.py --suite main \ + --trajectories runs/my-model/main/trial-1 \ + --output runs/my-model/main/trial-1/scores.json +``` + +Use `--suite augmented` for `data_v0.1_multi/v1.3_augmented`. The scorer reads +trajectories recursively and runs the checked-in task-specific `verify()` +functions. It does not trust a submitter's success flag or execute a verifier +from a path embedded in a submitted trajectory. + +- `SR_percent` is populated only when every expected task has one valid result + and no parsing/verification errors occurred. +- `observed_SR_percent` describes the scored subset only; it is not a full-suite + score. Missing tasks are listed explicitly. +- Duplicate tasks, malformed JSON, empty trajectories, and verifier errors are + reported as errors, not silently counted as task failures. The command exits + nonzero for errors or when no trajectories can be scored. +- Score each trial separately. Do not combine repeated attempts in one directory + or select only the best attempt. Record all trials and report the aggregation + protocol when submitting results. + +New runner outputs include `task_key` and `task_source`. Historical main logs with +only `task_id: task_NN` are supported. Augmented logs need an explicit task key or +source filename; an ID alone cannot distinguish the variants. + +The runner now selects adjacent augmented verifiers for early stopping, and +uses the same selection logic as this scorer. Older augmented runs may have +stopped using main-task criteria; rescore and review those trajectories rather +than assuming their saved success flags remain comparable. + +## Partial success rate (PSR) + +A validated, portable implementation reproducing the paper's PSR is not present +in this upstream checkout. The separately maintained local experiment copy has +a reconstruction, but it has not been cross-validated against the historical +implementation. Do not label a replacement calculation as the published metric. +Report PSR as unavailable until its implementation and protocol are established. + +## Causal contribution efficiency (CCE) + +The existing analysis implementation is +`analysis/scripts/compute_cce_v2.py`. It uses an LLM judge; successful trajectories +require model API calls. Install its optional dependency separately: + +```bash +python -m pip install -r analysis/requirements-cce.txt +export CCE_API_KEY='your-judge-api-key' +export CCE_BASE_URL='https://api.openai.com/v1' +export CCE_JUDGE_MODEL='gpt-4.1' +python analysis/scripts/compute_cce_v2.py \ + --trajectory runs/my-model/smoke/task_01_trajectory.json \ + --output runs/my-model/smoke/cce.json +``` + +This command retains the research script's CCE calculation. It reads +`metrics.task_verified_success` from the trajectory: verify that flag against the +matching task verifier before judging external or historical trajectories. A +failed trajectory receives zero CCE without judging its actions. Preserve judge +model/version, endpoint, prompt/protocol, and per-task output. Different judges +are different evaluation configurations, not interchangeable leaderboard scores. +The older `--model` / `--task` arguments remain for historical log layouts; +`AGENTWORLD_LOGS_DIR` overrides their log root. + +## Submit through a GitHub issue + +Use the website's [submission guide](https://agentworld.io/submission-guide) and +[Result submission issue form](https://github.com/openagents-org/agentworld-web/issues/new?template=result-submission.yml). +GitHub issues are the only result-submission route. Reviewers check the evidence +before results are published. + +Include model ID/version, harness code and commit, benchmark/task revision, +configuration and prompts, trials/seeds, coverage, metric definitions, scores, +and downloadable raw trajectories/logs with checksums. Remove credentials from +configuration files before sharing them. Identify custom harnesses, changed +budgets, and alternative judges explicitly. diff --git a/docs/benchmark/server.md b/docs/benchmark/server.md new file mode 100644 index 000000000..a97d54549 --- /dev/null +++ b/docs/benchmark/server.md @@ -0,0 +1,31 @@ +# Local game server + +The game runs from the TypeScript workspace in `packages/`; the Python benchmark +runner connects to its HTTP API. These are separate processes. + +From the repository root, install Node.js (the existing setup recommends Node 20) +and enable the repository's Yarn version through Corepack: + +```bash +corepack enable +yarn install +cp .env.defaults .env +yarn dev +``` + +Copy `.env.defaults` only for a new setup; retain an existing `.env`. Review its +ports before starting: the defaults are game port `7030`, HTTP API port `7031`, +and `SKIP_DATABASE=true`. MongoDB is optional for a disposable instance. Set +`SKIP_DATABASE=false` and configure MongoDB when persistence is needed. + +Set the runner's `AGENTWORLD_BASE_URL=http://localhost:7031` to match `API_PORT`. +A browser client URL or the public website API is not the game-control endpoint. +Use the HTTP API rather than the game's WebSocket port. + +For production builds, the existing commands are `yarn build` and `yarn start`. +The workspace's historical Node engine declaration differs from the Node 20 +recommendation; the onboarding checks do not certify a full game build across +Node versions. See [legacy setup](../legacy-usage.md) for the earlier engine docs. + +Use a dedicated local instance for benchmark runs. Concurrent task runs sharing +characters and world state can contaminate each other's observations and scores. diff --git a/docs/legacy-qwen-agent.md b/docs/legacy-qwen-agent.md new file mode 100644 index 000000000..6c1b55272 --- /dev/null +++ b/docs/legacy-qwen-agent.md @@ -0,0 +1,237 @@ +# Historical Qwen agent guide + +For current multi-provider setup, see [Benchmark quickstart](benchmark/quickstart.md). + +# Qwen AI Agent for Kaetram Game + +An intelligent AI agent that plays the Kaetram MMORPG game using Alibaba Cloud Qwen Function Calling capabilities. + +## Features + +- **OpenAI Compatible Function Calling**: Uses standard function calling format without XML dependencies +- **Clean Implementation**: No custom XML parsing - relies on native API function calling +- **Intelligent Gameplay**: AI makes strategic decisions about movement, combat, and resource collection +- **Interactive Mode**: Chat with the agent and give it specific instructions +- **Auto-Play Mode**: Let the agent play autonomously +- **Full Game API Coverage**: Supports all Kaetram AI agent APIs + +## Prerequisites + +1. **Kaetram Game Server**: Must be running on `http://localhost:9002` +2. **Alibaba Cloud Dashscope API Key**: Required for Qwen model access +3. **Python 3.8+**: Required for running the agent + +## Quick Start + +### 1. Install Dependencies + +```bash +cd agents/qwen +pip install -r requirements.txt +``` + +### 2. Configure API Key + +Set your Dashscope API key as an environment variable: + +```bash +export DASHSCOPE_API_KEY=your-actual-api-key +``` + +Or modify the `config.py` file directly. + +### 3. Start Kaetram Game Server + +Make sure the Kaetram game server is running with API enabled: + +```bash +# In the main Kaetram directory +yarn dev +``` + +The server should be accessible at `http://localhost:9002` + +### 4. Run the Agent + +#### Interactive Mode (Default) +```bash +python main.py +``` + +#### Auto-Play Mode +```bash +python main.py --auto +python main.py --auto --steps 20 # Run for 20 steps +``` + +#### Test Connections +```bash +python main.py --test +``` + +#### Run Examples +```bash +python example.py # Run all usage examples +python spawn_example.py # Demo automatic spawn position teleport +``` + +## Usage Examples + +### Interactive Mode Commands + +- `start` - Start a new game session (login/create character) +- `auto` - Switch to auto-play mode +- `reset` - Reset conversation history +- `quit` or `exit` - Exit the program + +### Sample Interactions + +``` +๐Ÿ‘ค You: start +๐Ÿค– Agent: [Logs into the game and starts playing] + +๐Ÿ‘ค You: explore the area and tell me what you see +๐Ÿค– Agent: [Observes environment and describes surroundings] + +๐Ÿ‘ค You: find some resources and collect them +๐Ÿค– Agent: [Searches for and collects available resources] + +๐Ÿ‘ค You: if you see any enemies, attack them +๐Ÿค– Agent: [Engages in combat if enemies are found] +``` + +## Configuration + +Edit `config.py` to customize: + +- **API Settings**: Qwen model, API endpoints +- **Game Settings**: Server URL, character credentials +- **Agent Behavior**: Combat enabled, chat enabled, exploration priority +- **Spawn Position**: Automatic teleport to specific coordinates after login + +### Spawn Position Configuration + +Configure automatic teleportation after login in `config.py`: + +```python +SPAWN_POSITION = { + "enabled": True, # Set to False to disable auto teleport + "x": 250, # X coordinate for spawn position + "y": 180, # Y coordinate for spawn position + "withAnimation": False # Whether to show teleport animation +} +``` + +When enabled, the agent will automatically teleport to the specified coordinates immediately after successful login or character creation. + +## Game Actions Supported + +The agent can perform all Kaetram AI actions: + +1. **Character Management** + - Create character + - Login character + +2. **Movement & Navigation** + - Move to coordinates + - Enter portals + - Stop movement + +3. **Environment Interaction** + - Observe surroundings + - Collect resources + - Target entities + +4. **Combat System** + - Target enemies + - Attack targets + - Stop combat + +5. **Equipment & Inventory** + - Equip items + - Manage inventory + +6. **Social Features** + - Send chat messages + - Interact with players + +## Architecture + +``` +agents/qwen/ +โ”œโ”€โ”€ config.py # Configuration settings +โ”œโ”€โ”€ game_tools.py # Kaetram API wrapper functions +โ”œโ”€โ”€ tool_definitions.py # Function calling tool definitions +โ”œโ”€โ”€ qwen_agent.py # Main AI agent implementation +โ”œโ”€โ”€ main.py # Entry point and CLI interface +โ”œโ”€โ”€ example.py # Usage examples and demos +โ”œโ”€โ”€ spawn_example.py # Spawn position demo +โ”œโ”€โ”€ requirements.txt # Python dependencies +โ””โ”€โ”€ README.md # This file +``` + +## How It Works + +1. **OpenAI Compatible Function Calling**: Uses standard OpenAI function calling format via tools parameter +2. **Clean Implementation**: No XML tags or custom parsing - relies on native API function calling +3. **API Integration**: Game tools wrap Kaetram's REST APIs with proper authentication +4. **Auto Spawn Positioning**: Automatically teleports to configured coordinates after successful login +5. **Strategic Decision Making**: The AI model makes intelligent decisions based on game state +6. **Continuous Learning**: The agent maintains conversation context for better decision making + +## Troubleshooting + +### Common Issues + +1. **API Key Error** + ``` + Error: DASHSCOPE_API_KEY not configured! + ``` + Solution: Set your Dashscope API key in environment variables + +2. **Game Server Connection Failed** + ``` + Game server connection failed: Connection refused + ``` + Solution: Ensure Kaetram server is running on localhost:9002 + +3. **Character Already Exists** + ``` + Failed to create character: Username already exists + ``` + Solution: The agent will automatically try to login instead + +4. **Token Errors** + ``` + Error: No token available. Please login first. + ``` + Solution: Run the 'start' command to login first + +### Debug Mode + +Enable debug logging by modifying `config.py`: + +```python +DEBUG_MODE = True +VERBOSE_LOGGING = True +``` + +## API Reference + +For detailed information about Kaetram's AI Agent APIs, see: +- [Kaetram AI API Documentation](../packages/server/src/network/README-AI-API.md) + +For Qwen Function Calling documentation, see: +- [Alibaba Cloud Qwen Function Calling](https://www.alibabacloud.com/help/zh/model-studio/qwen-function-calling) + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Test with both interactive and auto-play modes +5. Submit a pull request + +## License + +This project follows the same license as the main Kaetram project. \ No newline at end of file diff --git a/docs/legacy-usage.md b/docs/legacy-usage.md new file mode 100644 index 000000000..1f6c5b527 --- /dev/null +++ b/docs/legacy-usage.md @@ -0,0 +1,422 @@ +# Historical usage reference + +For the maintained onboarding path, see [Benchmark quickstart](benchmark/quickstart.md). +The examples below predate that guide; model names, paths, and configuration snippets may differ. + +# AgentWorld + +![MPL-2.0 License](https://img.shields.io/github/license/Kaetram/Kaetram-Open) [![Website](https://img.shields.io/website?url=https%3A%2F%2Fagentworld.io&style=flat)](https://agentworld.io) + +AgentWorld is a collaborative gaming environment where AI agents can interact, explore, and play together in a 2D multiplayer world. Built as a research platform for multi-agent AI systems, AgentWorld enables developers to create, control, and study AI agents in a rich gaming environment. + +Key features for AI research and development: +- **Multi-agent coordination**: Agents can collaborate, compete, and communicate with each other +- **Rich observation space**: Comprehensive API for agents to perceive their environment +- **Action space**: Agents can move, chat, fight, craft, and interact with the world +- **Persistent world**: Agents can learn and adapt over time in a persistent environment +- **Research-friendly**: Built-in APIs for easy integration with AI frameworks + +## Credits + +AgentWorld is built upon the excellent open-source foundation of [Kaetram](https://github.com/Kaetram/Kaetram-Open/tree/master), specifically based on the Kaetram master branch. Kaetram is an open-source RPG game that expands on Little Workshop's BrowserQuest. We're grateful to the Kaetram community for creating such a robust and well-designed game engine that serves as the perfect foundation for multi-agent AI research. + + +## Get Started + +### Prerequisites + +You must first [install Node.js](https://nodejs.org/en/download) to run the project, and +_optionally_ [install MongoDB](https://www.mongodb.com/try/download/community) to store user data on +the server. + +#### NOTE: Node.js + +> We recommend using Node.js version `v20` or higher, following the +> [Long Term Support (LTS) schedule](https://nodejs.org/en/about/releases), to have the most stable +> experience when developing/experimenting with AgentWorld. Older versions may not work with our +> current dependencies and package manager. + +#### NOTE: MongoDB + +> MongoDB is not a requirement for AgentWorld to run, but you can store and save user data if you +> install it and run an online environment with all the features enabled. To do this, see +> [Configuration](#configuration), and set `SKIP_DATABASE=false`. _If you do choose to install +> MongoDB, a user is not necessary, but you can enable authentication with the `MONGODB_AUTH` +> setting._ + +#### Yarn + +You will also need to enable [Yarn](https://yarnpkg.com) through [Corepack](https://nodejs.org/dist/latest/docs/api/corepack.html), to manage the dependencies. + +> The preferred way to manage Yarn is through [Corepack](https://nodejs.org/dist/latest/docs/api/corepack.html), a new binary shipped with all Node.js releases [...] +> +> To enable it, run the following command: +> +> ```console +> corepack enable +> ``` +> +> + +### Installing + +Install the dependencies by simply running + +```console +yarn +``` + +### Configuration + +Copy and rename [`.env.defaults`](../.env.defaults) to `.env`, and modify the contents to fit your +needs. + +```console +cp .env.defaults .env +``` + +Enable Mongodb by setting `SKIP_DATABASE=false` in the `.env` file. + + +### Running + + +To run live development builds, use + +```console +yarn dev +``` + +To create production builds, run + +```console +yarn build +``` + +Then, to run each production build, use + +```console +yarn start +``` + +We recommend to run first with `yarn dev` to start the server in development mode, so that you can see the changes you make to the code in real time. + + +## Run Agent Console + + +AgentWorld comes with a CLI tool to deploy an agent and interact with it using natural language. Make sure the game server is running, then in another terminal run: + +```console +python agents/console.py \ +--provider openai \ +--model gpt-4o \ +--username test1 \ +--password test1 +``` + +### Advanced Agent Configuration + +You can configure the agent's initial state using command line options: + +```console +# Set initial location and combat skills +python agents/console.py \ +--provider openai \ +--model gpt-4o \ +--username test1 \ +--password test1 \ +--location 300,200 \ +--combat-level-accuracy 45 \ +--combat-level-strength 50 \ +--combat-level-magic 30 + +# Start with equipped items and inventory +python agents/console.py \ +--provider openai \ +--model gpt-4o \ +--username test1 \ +--password test1 \ +--equipped-items bastardsword whitearmor ironhelmet:1:3 \ +--inventory-items healingpotion:10 firepotion:5 stick:20 + +# Full configuration example +python agents/console.py \ +--provider openai \ +--model gpt-4o \ +--username test1 \ +--password test1 \ +--location 250,180 \ +--combat-level-accuracy 60 \ +--combat-level-strength 60 \ +--combat-level-defense 50 \ +--combat-level-health 70 \ +--combat-level-magic 40 \ +--combat-level-archery 35 \ +--equipped-items bastardsword:1:2 whitearmor goldboots \ +--inventory-items healingpotion:20 firepotion:10 pythararrow:100 \ +--task "explore the world and fight strong enemies" +``` + +#### Initial State Options + +- `--location x,y`: Teleport to coordinates at startup (e.g., `--location 250,180`) +- `--combat-level-xxx N`: Set specific combat skill levels (1-120) + - Available skills: accuracy, strength, defense, health, magic, archery + - HP and MP are automatically restored to maximum after setting combat levels +- `--equipped-items item1 item2:count:enchant`: Set initial equipped items +- `--inventory-items item1:count item2:count:enchant`: Set initial inventory items + +#### Item Format + +- Basic: `itemkey` (e.g., `coppersword`) +- With count: `itemkey:count` (e.g., `healingpotion:10`) +- With enchantment: `itemkey:count:enchant` (e.g., `ironhelmet:1:3`) + +This will launch a console where you can interact with the agent. + +#### Cheat Commands + +You can use cheat commands both interactively and with the `--task` option: + +- `/teleport x y` or `/teleport spawn` - Teleport to coordinates +- `/equip itemkey [count] [enchant]` - Give and equip item +- `/setlevel level` - Set all combat skills to level +- `/give itemkey [count]` - Give items to inventory +- `/fullequip` - Give essential equipment set +- `/observe [radius]` - Show raw observation JSON data + +Examples: +```bash +# Interactive mode +๐ŸŽฎ What would you like to do? > /observe 32 + +# Single-task mode +python agents/console.py --task "/observe 32" --username test1 --password test1 +python agents/console.py --task "/teleport 300 200" --username test1 --password test1 +python agents/console.py --task "/setlevel 50" --username test1 --password test1 + +# Create fresh characters (no password needed) +python agents/console.py --new-character --username freshbot --task "/observe" +python agents/console.py --new-character --username testchar --location 300,200 --combat-level-strength 45 +``` + +#### New Character Creation + +Use `--new-character` to create fresh characters or reset existing ones: + +- **No password required** - The system handles authentication automatically +- **Fresh state** - Characters start with default level 1 stats and empty inventory +- **Automatic reset** - If username exists, it's reset to default state first +- **Works with all options** - Combine with location, combat levels, equipment, etc. + +The `/observe` command will display the complete environment observation data in JSON format. + +## Task Runner System + +AgentWorld includes a comprehensive task runner system that allows you to execute structured tasks from YAML configuration files. This is ideal for automated testing, benchmarking, and running complex multi-step scenarios. + +### Prerequisites for Task Runner + +Before using the task runner, ensure you have: + +1. **Game server running**: Start AgentWorld with `yarn dev` +2. **API keys configured**: Set up your LLM provider API keys as environment variables + +```bash +# For Qwen (Alibaba Cloud DashScope) +export DASHSCOPE_API_KEY=your-dashscope-api-key + +# For OpenAI +export OPENAI_API_KEY=sk-your-openai-key + +# For Anthropic Claude +export ANTHROPIC_API_KEY=sk-ant-your-claude-key + +# For DeepSeek +export DEEPSEEK_API_KEY=sk-your-deepseek-key +``` + +### Quick Start with Task Runner + +1. **Create a task configuration** (see `data_v0.1_solo/task_1.yaml` for example) +2. **Use an agent configuration** (see `agents/configs/qwen_agent.yaml`) +3. **Run the task**: + +```bash +# Run a single task +python3 agents/run.py --task data_v0.1_solo/task_1.yaml --agent agents/configs/qwen_agent.yaml --output logs/ + +# Run all tasks in a folder +python3 agents/run.py --task-folder data_v0.1_solo/ --agent agents/configs/qwen_agent.yaml --output logs/ +``` + +### Task Configuration Format + +Create YAML files that define structured tasks for your agents: + +```yaml +# Task definition +task: + name: "Smithing Task - Craft Heavy Sword" + description: "craft a heavy sword using smithing skills" + +# Task objectives +objectives: + primary: "Craft a heavy sword using smithing skills" + secondary: + - "Gather any missing materials if needed" + - "Use smithing station to craft the sword" + - "Verify successful crafting completion" + +# Agent configuration +agent_1: + username: "smith_test" + new_character: true + + location: + x: 88 + y: 33 + + skill_levels: + smithing: 15 + mining: 10 + + inventory_items: + - item: "ironbar" + count: 3 + - item: "hilt2" + count: 1 + + equipped_items: + - item: "smithinghammer" + count: 1 + enchant: 0 + +# Success criteria +success_criteria: + - "Heavy sword is successfully crafted" + +# Optional: Game context and limitations +max_action_steps: 100 +relevant_game_context: | + You are a smithing expert who knows how to use the forge and anvil effectively. +``` + +### Agent Configuration + +Agent configurations define the LLM provider settings and behavior: + +```yaml +# agents/configs/qwen_agent.yaml +agent: + name: "QwenAgent" + provider: "qwen" + + llm: + model: "qwen-plus" + api_key: null # Uses DASHSCOPE_API_KEY environment variable + base_url: "https://dashscope.aliyuncs.com/compatible-mode/v1" + max_tokens: 4096 + temperature: 0.7 + + game: + host: "http://localhost:7031" + timeout: 30 + max_retries: 3 + + # Custom system prompt template (optional) + system_prompt: | + You are an intelligent AI agent that plays the AgentWorld MMORPG game... + + === CURRENT ENVIRONMENT OBSERVATION === + {{observation}} + === END OBSERVATION === +``` + +### Key Features + +- **๐Ÿ“‹ Structured Task Definitions**: Define complex multi-step tasks in YAML +- **๐Ÿค– Multi-Agent Support**: Run tasks with multiple agents (agent_1, agent_2, etc.) +- **๐Ÿ”ง Configurable Agents**: Customize LLM provider, model, and system prompts +- **๐Ÿ“Š Comprehensive Logging**: Detailed logs saved to specified output folder +- **๐Ÿ“ˆ Execution Metrics**: Track duration, iterations, success rates +- **๐ŸŽฏ Batch Processing**: Run entire folders of tasks automatically + +### Output and Logging + +The task runner creates comprehensive logs in your specified output directory: + +``` +logs/ +โ”œโ”€โ”€ task_runner_20240115_143022.log # Main runner log +โ”œโ”€โ”€ agent_1_20240115_143022.json # Individual agent execution log +โ”œโ”€โ”€ task_summary_20240115_143022.json # Task execution summary +โ””โ”€โ”€ folder_summary_20240115_143022.json # Batch execution summary (if running folder) +``` + +### Advanced Usage + +```bash +# Run with different agent configurations +python3 agents/run.py --task task.yaml --agent configs/openai_agent.yaml --output results/ +python3 agents/run.py --task task.yaml --agent configs/claude_agent.yaml --output results/ + +# Run multiple tasks with comprehensive logging +python3 agents/run.py --task-folder experiments/ --agent configs/qwen_agent.yaml --output experiment_results/ +``` + +### Task Runner vs Console + +| Feature | Console (`console.py`) | Task Runner (`run.py`) | +|---------|----------------------|------------------------| +| **Use Case** | Interactive development, testing | Automated execution, benchmarking | +| **Input** | Command line arguments | YAML configuration files | +| **Output** | Console + optional JSON log | Structured logs + summaries | +| **Multi-agent** | Single agent per session | Multiple agents per task | +| **Reproducibility** | Manual setup each time | Fully reproducible from configs | +| **Batch Processing** | No | Yes (entire folders) | + +Choose **console.py** for interactive development and testing, and **run.py** for automated task execution and research experiments. + +### Troubleshooting + +#### Common Issues + +**API Key Errors (401 Unauthorized)** +``` +Error: API call failed: 401 Client Error: Unauthorized +``` +- Verify your API key is correctly set as an environment variable +- For Qwen: Check your DashScope API key format and permissions +- For other providers: Ensure the API key has sufficient credits/permissions + +**Game Server Connection Issues** +``` +Error: Game server not responding +``` +- Ensure AgentWorld is running with `yarn dev` +- Check that the game server is accessible at `http://localhost:7031` +- Verify no firewall is blocking the connection + +**Task Execution Failures** +- Check the detailed logs in the output directory +- Review the task summary JSON for specific error messages +- Ensure task configuration YAML syntax is valid + +#### Debug Commands + +```bash +# Test game server connectivity +curl http://localhost:7031/ai/observe + +# Test API key configuration +echo $DASHSCOPE_API_KEY # Should show your API key + +# Run with verbose logging +python3 agents/run.py --task task.yaml --agent config.yaml --output logs/ 2>&1 | tee debug.log +``` + +## License + +AgentWorld inherits the MPL-2.0 license from Kaetram. \ No newline at end of file diff --git a/docs/repository-map.md b/docs/repository-map.md new file mode 100644 index 000000000..aad3d9eb7 --- /dev/null +++ b/docs/repository-map.md @@ -0,0 +1,41 @@ +# Repository map + +## Supported onboarding path + +| Area | Start here | Role | +| --- | --- | --- | +| Benchmark execution | `agents/run.py` | Load tasks/configuration, initialize characters, schedule turns, record trajectories | +| Model/harness adapters | `agents/agent_factory.py`, `agents/base_agent.py` | Provider integration and shared agent contract | +| Game actions | `agents/game_tools.py`, `agents/tool_definitions.py` | HTTP client and tool schemas | +| Main task set | `data_v0.1_multi/v1.3_benchmark/` | 100 YAMLs, per-task verifiers, shared verifier utilities | +| Augmented task set | `data_v0.1_multi/v1.3_augmented/` | 200 YAML variants with adjacent verifiers | +| Offline SR reporting | `benchmarks/score.py` | Coverage, errors, per-task outcomes, complete-suite SR | +| CCE research code | `analysis/scripts/compute_cce_v2.py` | LLM judge calculation | +| Game engine | `packages/server/`, `packages/common/` | Game state, networking, shared definitions | +| Browser client | `packages/client/` | Play/debug UI; the leaderboard website is a separate repository | +| Additional engine components | `packages/hub/`, `packages/tools/`, `packages/e2e/` | Hub, asset tools, engine tests | +| Contributor examples/tests | `examples/agents/`, `tests/benchmark/` | Adapter example and offline onboarding regression tests | + +## Historical and specialized material + +The repository retains older datasets (`v1_benchmark`, `v1.2_benchmark`, rewritten +and augmented-generation directories), solo tasks, task-generation utilities, +`evaluate_results*`, research reports, and visualization files. They are useful +for historical work but are not additional suites to mix into the main score. + +`agents/run_*.sh` and many `scripts/run_*.sh` files are machine-specific experiment +launchers. `scripts/benchmark/` belongs to an older tooling layout. Prefer +`agents/run.py` for new experiments. `beam_verifier/` is a separate search-based +verification experiment. Root `evaluate_tasks.py` evaluates task design, not +model benchmark scores. `task_verifier.py`, `task_verifier1.py`, and +`batch_verify.sh` are legacy verification entry points; the new guide uses the +per-task verifiers through `agents/verification.py`. + +Existing task/code paths stay in place to preserve historical scripts. Put new +onboarding docs under `docs/benchmark/`, examples under `examples/agents/`, and +portable evaluation entry points under `benchmarks/`. Put generated runs under +`runs/` (ignored by Git), not among source files. Install the Python runner in +`.venv/` and keep private configs in `agents/configs/*.local.yaml`. + +The leaderboard and website live in +[openagents-org/agentworld-web](https://github.com/openagents-org/agentworld-web). diff --git a/examples/agents/custom_agent.py b/examples/agents/custom_agent.py new file mode 100644 index 000000000..058500a64 --- /dev/null +++ b/examples/agents/custom_agent.py @@ -0,0 +1,14 @@ +"""Runnable adapter example: retains the reference OpenAI-compatible policy. + +Replace _make_api_call and _extract_tool_calls for another model protocol. +Keep BaseAgent's one-action-per-turn execution and trajectory recording contract. +""" +from openai_agent import OpenAIAgent + + +class CustomAgent(OpenAIAgent): + def __init__(self, *, api_key, model, username, password, base_url, + dump_prompts, llm_params): + super().__init__(api_key=api_key, model=model, username=username, + password=password, base_url=base_url, + dump_prompts=dump_prompts, llm_params=llm_params) diff --git a/tests/benchmark/test_cce_entrypoint.py b/tests/benchmark/test_cce_entrypoint.py new file mode 100644 index 000000000..8b838c102 --- /dev/null +++ b/tests/benchmark/test_cce_entrypoint.py @@ -0,0 +1,40 @@ +"""CCE entry-point checks use no judge or game calls.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / 'analysis/scripts/compute_cce_v2.py' + + +@unittest.skipUnless(importlib.util.find_spec('openai'), 'Install analysis/requirements-cce.txt') +class CCEEntryTests(unittest.TestCase): + def test_import_requires_no_credentials_or_client(self): + with patch.dict(os.environ, {}, clear=True): + spec = importlib.util.spec_from_file_location('test_cce', SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertIsNone(module.client) + with self.assertRaisesRegex(ValueError, 'CCE_API_KEY'): + module.call_llm('unused') + + def test_explicit_trajectory_scores_failure_without_model_call(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'task_01_trajectory.json' + output = Path(tmp) / 'cce.json' + path.write_text(json.dumps({'task_id': 'task_01', 'task_definition': {}, + 'metrics': {'task_verified_success': False}, 'rounds': []})) + env = {k: v for k, v in os.environ.items() if not k.startswith('CCE_')} + result = subprocess.run([sys.executable, str(SCRIPT), '--trajectory', str(path), + '--output', str(output)], env=env, capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(output.read_text()) + self.assertIsNone(report['model']) + self.assertEqual(report['trajectory'], str(path)) + self.assertEqual(report['results'][0]['ce'], 0) diff --git a/tests/benchmark/test_onboarding.py b/tests/benchmark/test_onboarding.py new file mode 100644 index 000000000..cc455a679 --- /dev/null +++ b/tests/benchmark/test_onboarding.py @@ -0,0 +1,191 @@ +"""Offline checks for the portable benchmark entry points. No service calls.""" +import importlib.util +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'agents')) +sys.path.insert(0, str(ROOT / 'examples/agents')) +sys.path.insert(0, str(ROOT / 'benchmarks')) +from agent_factory import AgentFactory +from config_loader import load_agent_config +from run import TaskRunner +from verification import verifier_for_task, verify_trajectory +import score + +MAIN = ROOT / 'data_v0.1_multi/v1.3_benchmark' +AUG = ROOT / 'data_v0.1_multi/v1.3_augmented' + + +def trajectory(key='task_01_magic_staff', passed=True): + return {'task_id': 'task_01', 'task_key': key, 'rounds': [{'actions': [{ + 'agent_name': 'wizard', 'observation': {'inventory': {'items': [ + {'key': 'staff', 'count': 1}] if passed else []}}}]}], 'metrics': {}} + + +class ConfigTests(unittest.TestCase): + def test_example_env_substitution_keeps_yaml_structure(self): + values = {'MODEL_NAME': 'test-model', 'MODEL_API_KEY': 'key: #\\n${UNSET}', + 'MODEL_BASE_URL': 'http://localhost:8000/v1', + 'AGENTWORLD_BASE_URL': 'http://localhost:7031'} + with patch.dict(os.environ, values): + data = load_agent_config(ROOT / 'agents/configs/openai-compatible.example.yaml')['agent'] + self.assertEqual(data['llm']['api_key'], values['MODEL_API_KEY']) + self.assertEqual(data['game']['host'], values['AGENTWORLD_BASE_URL']) + + def test_missing_variable_fails_before_execution(self): + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(ValueError, 'MODEL_NAME'): + load_agent_config(ROOT / 'agents/configs/openai-compatible.example.yaml') + + def test_literal_legacy_config_unchanged(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'config.yaml' + path.write_text('agent:\n name: example\n llm:\n model: literal\n') + self.assertEqual(load_agent_config(path)['agent']['llm']['model'], 'literal') + + +class AdapterTests(unittest.TestCase): + def test_explicit_model_and_game_endpoints_are_separate(self): + with patch('requests.Session.request', side_effect=AssertionError('Network used')): + agent = AgentFactory.create_agent('openai', api_key='test', model='test', + base_url='http://game:7031', llm_params={'base_url': 'http://model:8000/v1/'}) + self.assertEqual(agent.base_url, 'http://model:8000/v1') + self.assertEqual(agent.game_tools.base_url, 'http://game:7031') + self.assertEqual(agent.session.headers['Authorization'], 'Bearer test') + self.assertNotIn('X-API-Key', agent.session.headers) + + def test_legacy_gateway_auth_preserved(self): + agent = AgentFactory.create_agent('openai', api_key='test') + self.assertEqual(agent.session.headers['X-API-Key'], 'test') + self.assertNotIn('Authorization', agent.session.headers) + + def test_custom_adapter_receives_config(self): + agent = AgentFactory.create_agent('custom', api_key='test', model='custom-model', + llm_params={'agent_class': 'custom_agent:CustomAgent', + 'base_url': 'http://model/v1', 'auth_header': 'X-API-Key'}) + self.assertEqual(type(agent).__name__, 'CustomAgent') + self.assertEqual(agent.model, 'custom-model') + self.assertEqual(agent.base_url, 'http://model/v1') + self.assertEqual(agent.session.headers['X-API-Key'], 'test') + + def test_invalid_custom_class_rejected(self): + for target in ['wrong', 'pathlib:Path']: + with self.subTest(target=target), self.assertRaises(ValueError): + AgentFactory.create_agent('custom', llm_params={'agent_class': target}) + + +class VerifierTests(unittest.TestCase): + def test_all_catalog_tasks_have_verifiers(self): + for directory, count in [(MAIN, 100), (AUG, 200)]: + tasks = list(directory.glob('*.yaml')) + self.assertEqual(len(tasks), count) + for task in tasks: + self.assertTrue(verifier_for_task(task).is_file(), task.name) + + def test_actual_main_verifier_positive_and_negative(self): + task = MAIN / 'task_01_magic_staff.yaml' + self.assertTrue(verify_trajectory(trajectory(), task)[0]) + self.assertFalse(verify_trajectory(trajectory(passed=False), task)[0]) + + def test_augmented_uses_adjacent_verifier(self): + for suffix in ['v1', 'v2']: + task = AUG / f'task_01_magic_staff_{suffix}.yaml' + self.assertEqual(verifier_for_task(task), task.with_name(task.stem + '_success_criteria.py')) + + def test_unknown_variant_never_falls_back_to_main(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError): + verifier_for_task(Path(tmp) / 'task_01_unknown_v2.yaml') + + def test_runner_preserves_variants_and_metadata(self): + runner = TaskRunner.__new__(TaskRunner) + runner.logger = Mock() + with tempfile.TemporaryDirectory() as tmp: + runner.output_dir = Path(tmp) + for suffix in ['v1', 'v2']: + key = f'task_01_magic_staff_{suffix}' + runner._init_trajectory(str(AUG / (key + '.yaml')), None) + self.assertEqual(runner.trajectory_data['task_key'], key) + self.assertEqual(runner.trajectory_data['task_source'], str(AUG / (key + '.yaml'))) + runner.trajectory_data['rounds'] = trajectory()['rounds'] + self.assertTrue(runner._check_task_success_early()[0]) + runner._save_trajectory(None, {}) + self.assertEqual(len(list(Path(tmp).glob('*_trajectory.json'))), 2) + runner._init_trajectory(str(MAIN / 'task_01_magic_staff.yaml'), None) + runner._save_trajectory(None, {}) + self.assertTrue((Path(tmp) / 'task_01_trajectory.json').exists()) + + +class ScoreTests(unittest.TestCase): + def write(self, directory, name, data): + path = directory / (name + '_trajectory.json') + path.write_text(json.dumps(data)) + return path + + def test_partial_coverage_not_published_as_full_score(self): + with tempfile.TemporaryDirectory() as tmp: + self.write(Path(tmp), 'one', trajectory()) + report = score.score(tmp, 'main') + self.assertFalse(report['complete']) + self.assertIsNone(report['SR_percent']) + self.assertEqual(report['observed_SR_percent'], 100) + self.assertEqual(len(report['missing_tasks']), 99) + + def test_complete_suite_uses_expected_denominator(self): + with tempfile.TemporaryDirectory() as tmp: + for i, task in enumerate(sorted(MAIN.glob('*.yaml'))): + self.write(Path(tmp), task.stem, trajectory(task.stem, passed=i < 50)) + with patch.object(score, 'verify_trajectory', side_effect=lambda t, p: ( + bool(t['rounds'][0]['actions'][0]['observation']['inventory']['items']), 'fixture')): + report = score.score(tmp, 'main') + self.assertTrue(report['complete']) + self.assertEqual(report['SR_percent'], 50) + + def test_duplicate_attempts_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + self.write(Path(tmp), 'one', trajectory()) + self.write(Path(tmp), 'two', trajectory()) + report = score.score(tmp, 'main') + self.assertIn('Duplicate', report['errors'][0]['error']) + self.assertIsNone(report['observed_SR_percent']) + + def test_ambiguous_augmented_legacy_id_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + data = trajectory(); del data['task_key'] + self.write(Path(tmp), 'one', data) + report = score.score(tmp, 'augmented') + self.assertIn('Ambiguous', report['errors'][0]['error']) + + def test_malformed_and_empty_trajectories_are_errors(self): + with tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / 'bad_trajectory.json').write_text('{') + data = trajectory(); data['rounds'] = [] + self.write(Path(tmp), 'empty', data) + report = score.score(tmp, 'main') + self.assertEqual(len(report['errors']), 2) + self.assertEqual(report['scored_tasks'], 0) + + def test_saved_absolute_source_does_not_execute_external_verifier(self): + with tempfile.TemporaryDirectory() as tmp: + data = trajectory(); del data['task_key'] + data['task_source'] = '/another-machine/task_01_magic_staff.yaml' + self.write(Path(tmp), 'one', data) + report = score.score(tmp, 'main') + self.assertEqual(report['successes'], 1) + self.assertEqual(report['errors'], []) + + def test_wrong_suite_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + self.write(Path(tmp), 'one', trajectory('task_01_magic_staff_v1')) + report = score.score(tmp, 'main') + self.assertIn('not in the selected suite', report['errors'][0]['error']) + + +if __name__ == '__main__': + unittest.main()